Adding in weaviate code

.codeclimate.yml

@@ -0,0 +1,11 @@
+version: "2"
+exclude_pattern:
+  - client/
+  - cmd/weaviate-server/main.go
+
+  # exclude all but the configure_weaviate.go file in restapi/,
+  # given that the other files are automatically generated.
+  - restapi/
+  - !restapi/configure_weaviate*
+
+  - test/

.dockerignore

@@ -0,0 +1,39 @@
+.git
+Dockerfile.*
+docker-compose.yml
+data*
+logs/
+snapshots/
+snapshots-fs/
+snapshots-node2/
+snapshots-s3/
+snapshots-gcs/
+backups/
+backups-fs/
+backups-node2/
+backups-s3/
+backups-gcs/
+test/
+tools/
+**/*.fvecs  # sift data
+**/*.png  # profiling
+**/*.out
+**/*.csv
+**/*.md
+**/*_test.go
+**/.DS_Store
+**/testdata/
+coverage-integration.txt
+coverage-unit.txt
+
+# wikipedia dataset when downloaded according to docs
+weaviate-wikipedia-*
+# the local path when extracting the wiki dataset
+var/weaviate/
+
+# IDE files
+.idea
+.vscode/
+
+# ignore archives, they are most likely part of some debugging and never part of actual code
+*.tar.gz

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# Ci related folders
+/.github/ @weaviate/core
+/ci/ @weaviate/core

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,9 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Need Help? Go to the community forum
+    url: https://forum.weaviate.io
+    about: "Community-powered knowledge repository: search, ask questions, give feedback"
+
+  - name: Want to chat? Go to the community slack
+    url: https://weaviate.io/slack
+    about: "Chat with the community about issues, questions, networking and feedback"

.github/ISSUE_TEMPLATE/create_issue.yml

@@ -0,0 +1,59 @@
+name: Found a bug?
+description: Report your bug here.
+labels: ["bug", "triage"]
+body:
+
+  - type: markdown
+    attributes:
+      value: |
+        #### Before you get started
+        * Check to make sure someone hasn't already opened a similar [issue](https://github.com/weaviate/weaviate-io/issues).
+        * Check this example of a [good bug report](https://github.com/weaviate/weaviate/issues/3762).
+        * Read the [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct).
+
+  - type: textarea
+    id: how_to_reproduce
+    attributes:
+      label: How to reproduce this bug?
+      description: Specify the steps here in order to reproduce this bug.
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: What is the expected behavior?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual_behavior
+    attributes:
+      label: What is the actual behavior?
+    validations:
+      required: true
+
+  - type: textarea
+    id: suporting_information
+    attributes:
+      label: Supporting information
+      description: Please, paste any logs, context information (client version? environment variables?) or other details in here.
+    validations:
+      required: false
+
+  - type: input
+    id: server_version
+    attributes:
+      label: Server Version
+      description: What Weaviate Version are you running?
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: Code of Conduct
+      description: This project has a Code of Conduct. All participants are expected to understand and follow the CoC.
+      options:
+        - label: I have read and agree to the Weaviate's [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct)
+          required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,29 @@
+name: Missing a Feature?
+description: Suggest it here.
+labels: ["feature request", "triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        #### Before you get started
+        * Is this feature already in our [roadmap](https://weaviate.io/developers/weaviate/roadmap)?
+        * Check to make sure someone hasn't already [requested this feature](https://github.com/weaviate/weaviate/issues?q=is:issue+label:"feature+request"). if that's the case, give it a thumbs up 👍
+        * Read the [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct).
+
+  - type: textarea
+    id: feature_request_body
+    attributes:
+      label: Describe your feature request
+      description: |
+        - Describe here your requested feature. Provide all necessary context, use cases and the reasoning on why this feature is a good one.
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: Code of Conduct
+      description: This project has a Code of Conduct that all participants are expected to understand and follow.
+      options:
+        - label: I have read and agree to the Weaviate's [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct)
+          required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+### What's being changed:
+
+
+### Review checklist
+
+- [ ] Documentation has been updated, if necessary. Link to changed documentation:
+- [ ] Chaos pipeline run or not necessary. Link to pipeline:
+- [ ] All new code is covered by tests where it is reasonable.
+- [ ] Performance tests have been run or not necessary.

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/stale.yml

@@ -0,0 +1,33 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 60
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+  - bug
+  - backlog
+  - needs-investigation
+# Label to use when marking an issue as stale
+staleLabel: autoclosed
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  Thank you for your contribution to Weaviate. This issue has not received any 
+  activity in a while and has therefore been marked as stale. Stale issues will
+  eventually be autoclosed. This does not mean that we are ruling out to work 
+  on this issue, but it most likely has not been prioritized high enough in the
+  last months.
+  
+  If you believe that this issue should remain open, please leave a short reply.
+  This lets us know that the issue is not abandoned and acts as a reminder for our
+  team to consider prioritizing this again.
+
+  Please also consider if you can make a contribution to help with the solution
+  of this issue. If you are willing to contribute, but don't know where to start,
+  please leave a quick message and we'll try to help you.
+
+  Thank you,
+  The Weaviate Team
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/branch.yaml

@@ -0,0 +1,33 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - '**'        # run on ever branch except master (is covered by PR)
+      - '!master'
+jobs:
+  Unit-Tests:
+    name: unit-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: Unit test
+        run: ./test/run.sh --unit-only
+  Integration-Tests:
+    name: integration-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: Integration test
+        run: ./test/run.sh --integration-only
+

.github/workflows/linter.yml

@@ -0,0 +1,29 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '**'
+  pull_request:
+jobs:
+  golangci:
+    name: golangci
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+      - uses: actions/checkout@v4
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.54
+          args: --timeout=5m
+  protolint:
+    name: protolint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: yoheimuta/action-protolint@v1

.github/workflows/pull_requests.yaml

@@ -0,0 +1,225 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '**'
+  pull_request:
+
+jobs:
+  Run-Swagger:
+    name: run-swagger
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: Run Swagger
+        run: ./tools/gen-code-from-swagger.sh
+      - name: Error on change
+        run: |
+          # check if anything is different
+          CHANGED=$(git status -s | wc -l)
+          if [ "$CHANGED" -gt 0 ]; then
+            echo "Please run ./tools/gen-code-from-swagger.sh script and commit changes:"
+            git status -s
+            exit 1
+          else
+            exit 0
+          fi
+  Vulnerability-Scanning:
+    name: vulnerability-scanning
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.head.repo.fork }}  # no PRs from fork
+    steps:
+      - uses: actions/checkout@v4
+      - name: Scan for Vulnerabilities in Code
+        uses: Templum/[email protected]
+        with:
+          go-version: '1.21'
+          package: ./...
+          fail-on-vuln: true
+  Unit-Tests:
+    name: unit-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: Unit test
+        run: ./test/run.sh --unit-only
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report-unit
+          path: coverage-unit.txt
+  Integration-Tests:
+    name: integration-tests  
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: Integration test
+        run: ./test/run.sh --integration-only
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report-integration
+          path: coverage-integration.txt
+  Modules-Acceptance-Tests:
+    name: modules-acceptance-tests
+    runs-on: ubuntu-latest-4-cores
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ["--modules-backup-only", "--modules-except-backup"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: Acceptance tests (modules)
+        run: ./test/run.sh ${{ matrix.test }}
+  Acceptance-Tests:
+    name: acceptance-tests  
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: Acceptance tests
+        env:
+          WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
+          WCS_DUMMY_CI_PW_2: ${{ secrets.WCS_DUMMY_CI_PW_2 }}
+        run: ./test/run.sh --acceptance-only
+  Codecov:
+    needs: [Unit-Tests, Integration-Tests]
+    name: codecov
+    runs-on: ubuntu-latest
+    if: ${{ (github.ref_type == 'branch') && (github.ref_name != 'master') }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download coverage artifacts integration
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage-report-unit
+      - name: Download coverage unit
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage-report-integration
+      - name: Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          fail_ci_if_error: false
+          files: ./coverage-integration.txt, ./coverage-unit.txt
+          verbose: true
+  Compile-and-upload-binaries:
+    name: compile-and-upload-binaries
+    runs-on: ubuntu-latest-4-cores
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: goreleaser
+        run: |
+          echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list
+          sudo apt update
+          sudo apt install goreleaser
+          GIT_HASH=$(git rev-parse --short HEAD) goreleaser build  --clean --snapshot
+      - name: Upload macos
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-macos-unsigned
+          path: dist/weaviate_darwin_all
+      - name: Upload windows
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-windows-amd64
+          path: dist/weaviate_windows_amd64_v1
+      - name: Upload windows
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-windows-arm64
+          path: dist/weaviate_windows_arm64
+      - name: Upload linux amd64
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-linux-amd64
+          path: dist/weaviate_linux_amd64_v1
+      - name: Upload linux arm64
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-linux-arm64
+          path: dist/weaviate_linux_arm64
+
+
+  Acceptance-Tests-windows:
+    name: acceptance-tests-windows
+    needs: Compile-and-upload-binaries
+    runs-on: windows-latest
+    env:
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: true
+      PERSISTENCE_DATA_PATH: /tmp
+      QUERY_DEFAULTS_LIMIT: 20
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: binaries-windows-amd64
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: true
+      - name: start weaviate
+        shell: bash
+        # Weaviate is started without a Vectorizer as running text2vec-contextionary on GH actions is difficult:
+        # - docker on GHA only supports windows container - which we currently are not build
+        # - building those containers without a windows machine is difficult to figure out
+        run: ./weaviate.exe --scheme http --port 8080 &
+      - name: run acceptance tests
+        shell: bash
+        run: go test -count 1 -race test/acceptance/actions/*.go  # tests that don't need a Vectorizer
+
+  Push-Docker:
+    needs: [Acceptance-Tests, Modules-Acceptance-Tests, Unit-Tests, Integration-Tests, Vulnerability-Scanning, Run-Swagger]
+    name: push-docker
+    runs-on: ubuntu-latest-8-cores
+    if: ${{ !github.event.pull_request.head.repo.fork }}  # no PRs from fork
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{secrets.DOCKER_USERNAME}}
+          password: ${{secrets.DOCKER_PASSWORD}}
+      - name: Push container
+        id: push-container
+        run: ./ci/push_docker.sh
+        env:
+          PR_TITLE: "${{ github.event.pull_request.title }}"
+      - name: Generate Report
+        env:
+          PREVIEW_TAG: "${{ steps.push-container.outputs.PREVIEW_TAG }}"
+        run: ./ci/generate_docker_report.sh

.github/workflows/release.yaml

@@ -0,0 +1,32 @@
+name: Generate release assets
+
+on:
+  release:
+    types: [published]
+
+env:
+  CGO_ENABLED: 0
+
+permissions:
+  contents: write
+
+jobs:
+  releases-matrix:
+    name: Release precompiled binaries
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux]
+        goarch: [amd64, arm64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: wangyoucao577/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          goversion: "1.21"
+          project_path: "./cmd/weaviate-server"
+          extra_files: LICENSE README.md
+          ldflags: -w -extldflags "-static" -X github.com/weaviate/weaviate/usecases/config.GitHash='"$GITHASH"'
+          sha256sum: true

.gitignore

@@ -0,0 +1,109 @@
+# ignore release dirs
+nightly/*
+dist/*
+dist
+nightly
+
+# ignore confectionary files
+contextionary/contextionary.vocab
+contextionary/contextionary.idx
+contextionary/contextionary.knn
+contextionary/stopwords.json
+wget-log*
+
+# ignore update files
+semver
+cc-test-reporter
+
+# Ignore OLD files
+*.OLD
+*.OLD.go
+
+# Do not include dist tmp
+dist_tmp
+
+# No weaviate bins
+./weaviate
+
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Contents of temp-directory
+temp
+.temp
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+node_modules
+
+# Optional npm cache directory
+.npm
+
+# Optional REPL history
+.node_repl_history
+
+# MAC stuff
+.DS_Store
+.!
+
+# IDE files
+.idea/
+.vscode/
+
+# Local files
+_local
+
+# Remove weave files
+weave-current.json
+weave-current-discovery.json
+
+# local nodes folders
+graphqlapi/prototype/node_modules/
+
+/data*/
+/snapshots*/
+/backups*/
+docker-compose/runtime-stable/data
+docker-compose/runtime-unstable/data
+c.out
+/cmd/weaviate-server/build_artifacts/
+
+esbackups/
+**/testdata/
+
+# coverage files
+coverage*.txt
+
+# go mod
+vendor/
+
+test/benchmark/benchmark_results.json
+# benchmark results files copied from a remote run
+benchmark_results_*.json
+
+# local credentials
+creds.json
+
+# VSCode
+cmd/weaviate-server/__debug_bin*

.golangci.yml

@@ -0,0 +1,62 @@
+linters:
+  disable:
+    - errcheck
+  enable:
+    - misspell
+    - bodyclose
+    - gofumpt
+    - exhaustive
+    - govet
+    - unused
+    - nolintlint
+linters-settings:
+  exhaustive:
+    # Presence of "default" case in switch statements satisfies exhaustiveness,
+    # even if all enum members are not listed.
+    default-signifies-exhaustive: true
+issues:
+  exclude-rules:
+    # Exclude some staticcheck messages
+    - linters:
+        - staticcheck
+      text: "S1034"
+    - linters:
+        - staticcheck
+      text: "SA1029:"
+    - linters:
+        - staticcheck
+      text: "SA1015:"
+    - linters:
+        - staticcheck
+      text: "SA5011"
+    - linters:
+        - govet
+      text: "composites"
+    - linters:
+        - staticcheck
+      path: adapters/handlers/grpc/v1/filters.go
+      text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API
+    - linters:
+        - staticcheck
+      path: adapters/handlers/grpc/v1/prepare_reply.go
+      text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API
+    - linters:
+        - staticcheck
+      path: adapters/handlers/grpc/v1/parse_search_request.go
+      text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API
+    - linters:
+        - staticcheck
+      path: adapters/handlers/grpc/v1/batch_parse_request.go
+      text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API
+    - linters:
+        - staticcheck
+      path: adapters/handlers/grpc/v1/service.go
+      text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API
+    - linters:
+        - staticcheck
+      path: test/acceptance/grpc/grpc_test_deprecated.go
+      text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API
+run:
+  build-tags:
+    - integrationTest
+    - integrationTestSlow

.goreleaser.yaml

@@ -0,0 +1,24 @@
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+      - windows
+    main: ./cmd/weaviate-server
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -w
+      - -extldflags "-static"
+      - -X github.com/weaviate/weaviate/usecases/config.GitHash={{ .Env.GIT_HASH }}
+
+# create a "fat" binary for MacOS
+universal_binaries:
+  - replace: true
+

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: local
+    hooks:
+      - id: gofumpt
+        name: gofumpt
+        entry: gofumpt -w
+        language: system
+        types: [ go ]
+      - id: swagger
+        name: swagger
+        entry: ./tools/gen-code-from-swagger.sh
+        language: system
+        types: [ go ]

.protolint.yaml

@@ -0,0 +1,4 @@
+lint:
+  rules_option:
+    max_line_length:
+      max_chars: 120

CITATION.cff

@@ -0,0 +1,23 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+title: "Weaviate"
+url: "https://github.com/weaviate/weaviate"
+authors:
+- family-names: "Dilocker"
+  given-names: "Etienne"
+- family-names: "van Luijt"
+  given-names: "Bob"
+  orcid: "https://orcid.org/0000-0002-8239-7421"
+- family-names: "Voorbach"
+  given-names: "Byron"
+- family-names: "Hasan"
+  given-names: "Mohd Shukri"
+- family-names: "Rodriguez"
+  given-names: "Abdel"
+- family-names: "Kulawiak"
+  given-names: "Dirk Alexander"
+  orcid: "https://orcid.org/0000-0002-9848-3818"
+- family-names: "Antas"
+  given-names: "Marcin"
+- family-names: "Duckworth"
+  given-names: "Parker"

CODE_OF_CONDUCT.md

@@ -0,0 +1,77 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [email protected]. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
+

CONTRIBUTING.md

@@ -0,0 +1,43 @@
+## Thanks for looking into contributing to Weaviate!
+We really appreciate that you are willing to spend some time and effort to make Weaviate better for everyone! 
+
+We have a detailed [contributor guide](https://weaviate.io/developers/contributor-guide/current/) as a part of our documentation. If you are new, we recommend reading through the [getting started guide for contributors](https://weaviate.io/developers/contributor-guide/current/getting-started/index.html) after reading this overview.
+
+## Finding a good issue to get started
+We use the `good-first-issue` labels on issues that we think are great to get started. Typically, they are issues that are isolated to a specific area of Weaviate, e.g. only the API layer or only a specific package.
+
+## What do you need to contribute
+To contribute to Weaviate Core you should have at least basic Golang skills. It is also tremendously helpful if you have successfully used Weaviate before, so that you have a rough understanding of the effects of changes etc. Any PR must include tests. We understand that not everyone is very familiar with testing strategies, and how we have setup our tests. This [guide on testing](https://weaviate.io/developers/contributor-guide/current/weaviate-core/tests.html) should be a great start. If you have completed a change, but are struggling with the test, you can also open a Draft PR and ask someone from our team for help. 
+
+## If in doubt, ask.
+The Weaviate team consists of some of the nicest people on this planet. If something is unclear or you'd like a second opinion, please don't hesitate to ask. We are glad that you want to help us, so naturally we will also do our best to help you on this journey
+
+## Development Environment
+*Please note that the Weaviate team uses Linux and Mac (darwin/arm64) machines exclusively. Development on Windows may lead to unexpected issues.*
+
+Here are some guides to get started in no time:
+* [Development Setup](https://weaviate.io/developers/contributor-guide/current/weaviate-core/setup.html)
+* [Code Structure](https://weaviate.io/developers/contributor-guide/current/weaviate-core/structure.html)
+* [CI/CD](https://weaviate.io/developers/contributor-guide/current/weaviate-core/cicd.html)
+* [Testing Philosophy](https://weaviate.io/developers/contributor-guide/current/weaviate-core/tests.html)
+
+## Tagging your commit
+Please tag your commit(s) with the appropriate GH issue that your change refers to, e.g. `gh-9001 reduce memory allocations of ACME widget`. Please also include something in your PR description to indicate which issue it will close, e.g. `fixes #9001` or `closes #9001`.
+
+## Pull Request
+If you open an external pull request our CI pipeline will get started. This external run will not have access to secrets. This prevents people from submitting a malicious PR to steal secrets. As a result the CI run will be slightly different from an internal one. For example, it will not automatically push a Docker image. If your PR is merged, a container with your changes will be built from the trunk. 
+
+## Agreements 
+
+### Code of Conduct
+Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. 
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](CODE_OF_CONDUCT.md)
+
+### Contributor License Agreement
+
+Contributions to Weaviate must be accompanied by a Contributor License Agreement. You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of Weaviate. Go to [this page](https://www.semi.technology/playbooks/misc/contributor-license-agreement.html) to read the current agreement.
+
+The process works as follows:
+
+- You contribute by opening a [pull request](#pull-request).
+- If you have not contributed before, our bot will ask to agree with the CLA

DockerfileHld

@@ -0,0 +1,49 @@
+# Dockerfile for development purposes.
+# Read docs/development.md for more information
+# vi: ft=dockerfile
+
+###############################################################################
+# Base build image
+FROM golang:1.21-alpine AS build_base
+RUN apk add bash ca-certificates git gcc g++ libc-dev
+WORKDIR /go/src/github.com/weaviate/weaviate
+ENV GO111MODULE=on
+# Populate the module cache based on the go.{mod,sum} files.
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+###############################################################################
+# This image builds the weaviate server
+FROM build_base AS server_builder
+ARG TARGETARCH
+ARG GITHASH="unknown"
+ARG EXTRA_BUILD_ARGS=""
+COPY . .
+RUN GOOS=linux GOARCH=$TARGETARCH go build $EXTRA_BUILD_ARGS \
+      -ldflags '-w -extldflags "-static" -X github.com/weaviate/weaviate/usecases/config.GitHash='"$GITHASH"'' \
+      -o /weaviate-server ./cmd/weaviate-server
+
+###############################################################################
+# This creates an image that can be used to fake an api for telemetry acceptance test purposes
+FROM build_base AS telemetry_mock_api
+COPY . .
+ENTRYPOINT ["./tools/dev/telemetry_mock_api.sh"]
+
+###############################################################################
+# This image gets grpc health check probe
+FROM build_base AS grpc_health_probe_builder
+ARG TARGETARCH
+RUN GRPC_HEALTH_PROBE_VERSION=v0.4.22 && \
+      wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} && \
+      chmod +x /bin/grpc_health_probe
+
+###############################################################################
+# Weaviate (no differentiation between dev/test/prod - 12 factor!)
+FROM alpine AS weaviate
+ENTRYPOINT ["/bin/weaviate"]
+COPY --from=grpc_health_probe_builder /bin/grpc_health_probe /bin/
+COPY --from=server_builder /weaviate-server /bin/weaviate
+RUN apk add --no-cache --upgrade ca-certificates openssl
+RUN mkdir ./modules
+CMD [ "--host", "0.0.0.0", "--port", "8080", "--scheme", "http"]

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2020-2024, Weaviate B.V.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

adapters/clients/client.go

@@ -0,0 +1,96 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+type retryClient struct {
+	client *http.Client
+	*retryer
+}
+
+func (c *retryClient) doWithCustomMarshaller(timeout time.Duration,
+	req *http.Request, body []byte, decode func([]byte) error,
+) (err error) {
+	ctx, cancel := context.WithTimeout(req.Context(), timeout)
+	defer cancel()
+	try := func(ctx context.Context) (bool, error) {
+		if body != nil {
+			req.Body = io.NopCloser(bytes.NewReader(body))
+		}
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+
+		respBody, err := io.ReadAll(res.Body)
+		if err != nil {
+			return shouldRetry(res.StatusCode), fmt.Errorf("read response: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusOK {
+			return shouldRetry(code), fmt.Errorf("status code: %v, error: %s", code, respBody)
+		}
+
+		if err := decode(respBody); err != nil {
+			return false, fmt.Errorf("unmarshal response: %w", err)
+		}
+
+		return false, nil
+	}
+	return c.retry(ctx, 9, try)
+}
+
+type retryer struct {
+	minBackOff  time.Duration
+	maxBackOff  time.Duration
+	timeoutUnit time.Duration
+}
+
+func newRetryer() *retryer {
+	return &retryer{
+		minBackOff:  time.Millisecond * 250,
+		maxBackOff:  time.Second * 30,
+		timeoutUnit: time.Second, // used by unit tests
+	}
+}
+
+func (r *retryer) retry(ctx context.Context, n int, work func(context.Context) (bool, error)) error {
+	delay := r.minBackOff
+	for {
+		keepTrying, err := work(ctx)
+		if !keepTrying || n < 1 || err == nil {
+			return err
+		}
+
+		n--
+		if delay = backOff(delay); delay > r.maxBackOff {
+			delay = r.maxBackOff
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return fmt.Errorf("%v: %w", err, ctx.Err())
+		case <-timer.C:
+		}
+		timer.Stop()
+	}
+}

adapters/clients/cluster_backups.go

@@ -0,0 +1,178 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/weaviate/weaviate/usecases/backup"
+)
+
+const (
+	pathCanCommit = "/backups/can-commit"
+	pathCommit    = "/backups/commit"
+	pathStatus    = "/backups/status"
+	pathAbort     = "/backups/abort"
+)
+
+type ClusterBackups struct {
+	client *http.Client
+}
+
+func NewClusterBackups(client *http.Client) *ClusterBackups {
+	return &ClusterBackups{client: client}
+}
+
+func (c *ClusterBackups) CanCommit(ctx context.Context,
+	host string, req *backup.Request,
+) (*backup.CanCommitResponse, error) {
+	url := url.URL{Scheme: "http", Host: host, Path: pathCanCommit}
+
+	b, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("marshal can-commit request: %w", err)
+	}
+
+	httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b))
+	if err != nil {
+		return nil, fmt.Errorf("new can-commit request: %w", err)
+	}
+
+	respBody, statusCode, err := c.do(httpReq)
+	if err != nil {
+		return nil, fmt.Errorf("can-commit request: %w", err)
+	}
+
+	if statusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code %d (%s)",
+			statusCode, respBody)
+	}
+
+	var resp backup.CanCommitResponse
+	err = json.Unmarshal(respBody, &resp)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal can-commit response: %w", err)
+	}
+
+	return &resp, nil
+}
+
+func (c *ClusterBackups) Commit(ctx context.Context,
+	host string, req *backup.StatusRequest,
+) error {
+	url := url.URL{Scheme: "http", Host: host, Path: pathCommit}
+
+	b, err := json.Marshal(req)
+	if err != nil {
+		return fmt.Errorf("marshal commit request: %w", err)
+	}
+
+	httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b))
+	if err != nil {
+		return fmt.Errorf("new commit request: %w", err)
+	}
+
+	respBody, statusCode, err := c.do(httpReq)
+	if err != nil {
+		return fmt.Errorf("commit request: %w", err)
+	}
+
+	if statusCode != http.StatusCreated {
+		return fmt.Errorf("unexpected status code %d (%s)",
+			statusCode, respBody)
+	}
+
+	return nil
+}
+
+func (c *ClusterBackups) Status(ctx context.Context,
+	host string, req *backup.StatusRequest,
+) (*backup.StatusResponse, error) {
+	url := url.URL{Scheme: "http", Host: host, Path: pathStatus}
+
+	b, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("marshal status request: %w", err)
+	}
+
+	httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b))
+	if err != nil {
+		return nil, fmt.Errorf("new status request: %w", err)
+	}
+
+	respBody, statusCode, err := c.do(httpReq)
+	if err != nil {
+		return nil, fmt.Errorf("status request: %w", err)
+	}
+
+	if statusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code %d (%s)",
+			statusCode, respBody)
+	}
+
+	var resp backup.StatusResponse
+	err = json.Unmarshal(respBody, &resp)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal status response: %w", err)
+	}
+
+	return &resp, nil
+}
+
+func (c *ClusterBackups) Abort(_ context.Context,
+	host string, req *backup.AbortRequest,
+) error {
+	url := url.URL{Scheme: "http", Host: host, Path: pathAbort}
+
+	b, err := json.Marshal(req)
+	if err != nil {
+		return fmt.Errorf("marshal abort request: %w", err)
+	}
+
+	httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b))
+	if err != nil {
+		return fmt.Errorf("new abort request: %w", err)
+	}
+
+	respBody, statusCode, err := c.do(httpReq)
+	if err != nil {
+		return fmt.Errorf("abort request: %w", err)
+	}
+
+	if statusCode != http.StatusNoContent {
+		return fmt.Errorf("unexpected status code %d (%s)",
+			statusCode, respBody)
+	}
+
+	return nil
+}
+
+func (c *ClusterBackups) do(req *http.Request) (body []byte, statusCode int, err error) {
+	httpResp, err := c.client.Do(req)
+	if err != nil {
+		return nil, 0, fmt.Errorf("make request: %w", err)
+	}
+
+	body, err = io.ReadAll(httpResp.Body)
+	if err != nil {
+		return nil, httpResp.StatusCode, fmt.Errorf("read response: %w", err)
+	}
+	defer httpResp.Body.Close()
+
+	return body, httpResp.StatusCode, nil
+}

adapters/clients/cluster_classifications.go

@@ -0,0 +1,127 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/pkg/errors"
+	"github.com/weaviate/weaviate/usecases/cluster"
+)
+
+type ClusterClassifications struct {
+	client *http.Client
+}
+
+func NewClusterClassifications(httpClient *http.Client) *ClusterClassifications {
+	return &ClusterClassifications{client: httpClient}
+}
+
+func (c *ClusterClassifications) OpenTransaction(ctx context.Context, host string,
+	tx *cluster.Transaction,
+) error {
+	path := "/classifications/transactions/"
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: host, Path: path}
+
+	pl := txPayload{
+		Type:    tx.Type,
+		ID:      tx.ID,
+		Payload: tx.Payload,
+	}
+
+	jsonBytes, err := json.Marshal(pl)
+	if err != nil {
+		return errors.Wrap(err, "marshal transaction payload")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(jsonBytes))
+	if err != nil {
+		return errors.Wrap(err, "open http request")
+	}
+
+	req.Header.Set("content-type", "application/json")
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusCreated {
+		if res.StatusCode == http.StatusConflict {
+			return cluster.ErrConcurrentTransaction
+		}
+
+		body, _ := io.ReadAll(res.Body)
+		return errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	return nil
+}
+
+func (c *ClusterClassifications) AbortTransaction(ctx context.Context, host string,
+	tx *cluster.Transaction,
+) error {
+	path := "/classifications/transactions/" + tx.ID
+	method := http.MethodDelete
+	url := url.URL{Scheme: "http", Host: host, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return errors.Wrap(err, "open http request")
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusNoContent {
+		return errors.Errorf("unexpected status code %d", res.StatusCode)
+	}
+
+	return nil
+}
+
+func (c *ClusterClassifications) CommitTransaction(ctx context.Context, host string,
+	tx *cluster.Transaction,
+) error {
+	path := "/classifications/transactions/" + tx.ID + "/commit"
+	method := http.MethodPut
+	url := url.URL{Scheme: "http", Host: host, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return errors.Wrap(err, "open http request")
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusNoContent {
+		return errors.Errorf("unexpected status code %d", res.StatusCode)
+	}
+
+	return nil
+}

adapters/clients/cluster_node.go

@@ -0,0 +1,66 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+
+	enterrors "github.com/weaviate/weaviate/entities/errors"
+	"github.com/weaviate/weaviate/entities/models"
+)
+
+type RemoteNode struct {
+	client *http.Client
+}
+
+func NewRemoteNode(httpClient *http.Client) *RemoteNode {
+	return &RemoteNode{client: httpClient}
+}
+
+func (c *RemoteNode) GetNodeStatus(ctx context.Context, hostName, className, output string) (*models.NodeStatus, error) {
+	p := "/nodes/status"
+	if className != "" {
+		p = path.Join(p, className)
+	}
+	method := http.MethodGet
+	params := url.Values{"output": []string{output}}
+	url := url.URL{Scheme: "http", Host: hostName, Path: p, RawQuery: params.Encode()}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return nil, enterrors.NewErrOpenHttpRequest(err)
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return nil, enterrors.NewErrSendHttpRequest(err)
+	}
+
+	defer res.Body.Close()
+	body, _ := io.ReadAll(res.Body)
+	if res.StatusCode != http.StatusOK {
+		return nil, enterrors.NewErrUnexpectedStatusCode(res.StatusCode, body)
+	}
+
+	var nodeStatus models.NodeStatus
+	err = json.Unmarshal(body, &nodeStatus)
+	if err != nil {
+		return nil, enterrors.NewErrUnmarshalBody(err)
+	}
+
+	return &nodeStatus, nil
+}

adapters/clients/cluster_schema.go

@@ -0,0 +1,164 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/weaviate/weaviate/usecases/cluster"
+)
+
+type ClusterSchema struct {
+	client *http.Client
+}
+
+func NewClusterSchema(httpClient *http.Client) *ClusterSchema {
+	return &ClusterSchema{client: httpClient}
+}
+
+func (c *ClusterSchema) OpenTransaction(ctx context.Context, host string,
+	tx *cluster.Transaction,
+) error {
+	path := "/schema/transactions/"
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: host, Path: path}
+
+	pl := txPayload{
+		Type:          tx.Type,
+		ID:            tx.ID,
+		Payload:       tx.Payload,
+		DeadlineMilli: tx.Deadline.UnixMilli(),
+	}
+
+	jsonBytes, err := json.Marshal(pl)
+	if err != nil {
+		return fmt.Errorf("marshal transaction payload: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(jsonBytes))
+	if err != nil {
+		return fmt.Errorf("open http request: %w", err)
+	}
+
+	req.Header.Set("content-type", "application/json")
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("send http request: %w", err)
+	}
+
+	defer res.Body.Close()
+	body, _ := io.ReadAll(res.Body)
+	if res.StatusCode != http.StatusCreated {
+		if res.StatusCode == http.StatusConflict {
+			return cluster.ErrConcurrentTransaction
+		}
+
+		return fmt.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	// optional for backward-compatibility before v1.17 where only
+	// write-transactions where supported. They had no return value other than
+	// the status code. With the introduction of read-transactions it is now
+	// possible to return the requested value
+	if len(body) == 0 {
+		return nil
+	}
+
+	var txRes txResponsePayload
+	err = json.Unmarshal(body, &txRes)
+	if err != nil {
+		return fmt.Errorf("unexpected error unmarshalling tx response: %w", err)
+	}
+
+	if tx.ID != txRes.ID {
+		return fmt.Errorf("unexpected mismatch between outgoing and incoming tx ids:"+
+			"%s vs %s", tx.ID, txRes.ID)
+	}
+
+	tx.Payload = txRes.Payload
+
+	return nil
+}
+
+func (c *ClusterSchema) AbortTransaction(ctx context.Context, host string,
+	tx *cluster.Transaction,
+) error {
+	path := "/schema/transactions/" + tx.ID
+	method := http.MethodDelete
+	url := url.URL{Scheme: "http", Host: host, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return fmt.Errorf("open http request: %w", err)
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("send http request: %w", err)
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusNoContent {
+		errBody, _ := io.ReadAll(res.Body)
+		return fmt.Errorf("unexpected status code %d: %s", res.StatusCode, errBody)
+	}
+
+	return nil
+}
+
+func (c *ClusterSchema) CommitTransaction(ctx context.Context, host string,
+	tx *cluster.Transaction,
+) error {
+	path := "/schema/transactions/" + tx.ID + "/commit"
+	method := http.MethodPut
+	url := url.URL{Scheme: "http", Host: host, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return fmt.Errorf("open http request: %w", err)
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("send http request: %w", err)
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusNoContent {
+		errBody, _ := io.ReadAll(res.Body)
+		return fmt.Errorf("unexpected status code %d: %s", res.StatusCode, errBody)
+	}
+
+	return nil
+}
+
+type txPayload struct {
+	Type          cluster.TransactionType `json:"type"`
+	ID            string                  `json:"id"`
+	Payload       interface{}             `json:"payload"`
+	DeadlineMilli int64                   `json:"deadlineMilli"`
+}
+
+type txResponsePayload struct {
+	Type    cluster.TransactionType `json:"type"`
+	ID      string                  `json:"id"`
+	Payload json.RawMessage         `json:"payload"`
+}

adapters/clients/cluster_schema_test.go

@@ -0,0 +1,441 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/weaviate/weaviate/usecases/cluster"
+)
+
+func TestOpenTransactionNoReturnPayload(t *testing.T) {
+	// The No-Return-Payload is the situation that existed prior to v1.17 where
+	// the only option for transactions was to broadcast updates. By keeping this
+	// test around, we can make sure that we are not breaking backward
+	// compatibility.
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		body, err := io.ReadAll(r.Body)
+		require.Nil(t, err)
+
+		var pl txPayload
+		err = json.Unmarshal(body, &pl)
+		require.Nil(t, err)
+
+		assert.Equal(t, "mamma-mia-paylodia-belissima", pl.Payload.(string))
+		w.WriteHeader(http.StatusCreated)
+	}
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	u, _ := url.Parse(server.URL)
+
+	c := NewClusterSchema(&http.Client{})
+
+	tx := &cluster.Transaction{
+		ID:      "12345",
+		Type:    "the best",
+		Payload: "mamma-mia-paylodia-belissima",
+	}
+
+	err := c.OpenTransaction(context.Background(), u.Host, tx)
+	assert.Nil(t, err)
+}
+
+func TestOpenTransactionWithReturnPayload(t *testing.T) {
+	// Newly added as part of v1.17 where read-transactions were introduced which
+	// are used to sync up cluster schema state when nodes join
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		resTx := txPayload{
+			Type:    "my-tx",
+			ID:      "987",
+			Payload: "gracie-mille-per-payload",
+		}
+		resBody, err := json.Marshal(resTx)
+		require.Nil(t, err)
+
+		w.WriteHeader(http.StatusCreated)
+		w.Write(resBody)
+	}
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	u, _ := url.Parse(server.URL)
+
+	c := NewClusterSchema(&http.Client{})
+
+	txIn := &cluster.Transaction{
+		ID:   "987",
+		Type: "my-tx",
+	}
+
+	err := c.OpenTransaction(context.Background(), u.Host, txIn)
+	assert.Nil(t, err)
+
+	expectedTxOut := &cluster.Transaction{
+		ID:      "987",
+		Type:    "my-tx",
+		Payload: json.RawMessage("\"gracie-mille-per-payload\""),
+	}
+
+	assert.Equal(t, expectedTxOut, txIn)
+}
+
+func TestOpenTransactionWithTTL(t *testing.T) {
+	deadline, err := time.Parse(time.RFC3339Nano, "2040-01-02T15:04:05.00Z")
+	require.Nil(t, err)
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		body, err := io.ReadAll(r.Body)
+		require.Nil(t, err)
+
+		var pl txPayload
+		err = json.Unmarshal(body, &pl)
+		require.Nil(t, err)
+
+		parsedDL := time.UnixMilli(pl.DeadlineMilli)
+		assert.Equal(t, deadline.UnixNano(), parsedDL.UnixNano())
+		w.WriteHeader(http.StatusCreated)
+	}
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	u, _ := url.Parse(server.URL)
+
+	c := NewClusterSchema(&http.Client{})
+
+	tx := &cluster.Transaction{
+		ID:       "12345",
+		Type:     "the best",
+		Payload:  "mamma-mia-paylodia-belissima",
+		Deadline: deadline,
+	}
+
+	err = c.OpenTransaction(context.Background(), u.Host, tx)
+	assert.Nil(t, err)
+}
+
+func TestOpenTransactionUnhappyPaths(t *testing.T) {
+	type test struct {
+		name                string
+		handler             http.HandlerFunc
+		expectedErr         error
+		expectedErrContains string
+		ctx                 context.Context
+		shutdownPrematurely bool
+	}
+
+	expiredCtx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tests := []test{
+		{
+			name: "concurrent transaction",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				defer r.Body.Close()
+				w.WriteHeader(http.StatusConflict)
+			},
+			expectedErr: cluster.ErrConcurrentTransaction,
+		},
+		{
+			name: "arbitrary 500",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				defer r.Body.Close()
+				w.WriteHeader(http.StatusInternalServerError)
+				w.Write([]byte("nope!"))
+			},
+			expectedErrContains: "nope!",
+		},
+		{
+			name: "invalid json",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				defer r.Body.Close()
+				w.WriteHeader(http.StatusCreated)
+				w.Write([]byte("<<<!@#*)!@#****@!''"))
+			},
+			expectedErrContains: "error unmarshalling",
+		},
+		{
+			name: "expired ctx",
+			ctx:  expiredCtx,
+			handler: func(w http.ResponseWriter, r *http.Request) {
+			},
+			expectedErrContains: "context",
+		},
+		{
+			name:                "remote server shut down",
+			shutdownPrematurely: true,
+			handler: func(w http.ResponseWriter, r *http.Request) {
+			},
+			expectedErrContains: "refused",
+		},
+		{
+			name: "tx id mismatch",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				resTx := txPayload{
+					Type:    "wrong-tx-id",
+					ID:      "987",
+					Payload: "gracie-mille-per-payload",
+				}
+				resBody, err := json.Marshal(resTx)
+				require.Nil(t, err)
+
+				w.WriteHeader(http.StatusCreated)
+				w.Write(resBody)
+			},
+			expectedErrContains: "mismatch between outgoing and incoming tx ids",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(test.handler))
+			if test.shutdownPrematurely {
+				server.Close()
+			} else {
+				defer server.Close()
+			}
+
+			u, _ := url.Parse(server.URL)
+
+			c := NewClusterSchema(&http.Client{})
+
+			tx := &cluster.Transaction{
+				ID:      "12345",
+				Type:    "the best",
+				Payload: "mamma-mia-paylodia-belissima",
+			}
+
+			if test.ctx == nil {
+				test.ctx = context.Background()
+			}
+
+			err := c.OpenTransaction(test.ctx, u.Host, tx)
+			assert.NotNil(t, err)
+
+			if test.expectedErr != nil {
+				assert.Equal(t, test.expectedErr, err)
+			}
+
+			if test.expectedErrContains != "" {
+				assert.Contains(t, err.Error(), test.expectedErrContains)
+			}
+		})
+	}
+}
+
+func TestAbortTransaction(t *testing.T) {
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		w.WriteHeader(http.StatusNoContent)
+	}
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	u, _ := url.Parse(server.URL)
+
+	c := NewClusterSchema(&http.Client{})
+
+	tx := &cluster.Transaction{
+		ID:      "am-i-going-to-be-cancelled",
+		Type:    "the worst",
+		Payload: "",
+	}
+
+	err := c.AbortTransaction(context.Background(), u.Host, tx)
+	assert.Nil(t, err)
+}
+
+func TestAbortTransactionUnhappyPaths(t *testing.T) {
+	type test struct {
+		name                string
+		handler             http.HandlerFunc
+		expectedErr         error
+		expectedErrContains string
+		ctx                 context.Context
+		shutdownPrematurely bool
+	}
+
+	expiredCtx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tests := []test{
+		{
+			name: "arbitrary 500",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				defer r.Body.Close()
+				w.WriteHeader(http.StatusInternalServerError)
+				w.Write([]byte("nope!"))
+			},
+			expectedErrContains: "nope!",
+		},
+		{
+			name: "expired ctx",
+			ctx:  expiredCtx,
+			handler: func(w http.ResponseWriter, r *http.Request) {
+			},
+			expectedErrContains: "context",
+		},
+		{
+			name:                "remote server shut down",
+			shutdownPrematurely: true,
+			handler: func(w http.ResponseWriter, r *http.Request) {
+			},
+			expectedErrContains: "refused",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(test.handler))
+			if test.shutdownPrematurely {
+				server.Close()
+			} else {
+				defer server.Close()
+			}
+
+			u, _ := url.Parse(server.URL)
+
+			c := NewClusterSchema(&http.Client{})
+
+			tx := &cluster.Transaction{
+				ID:      "12345",
+				Type:    "the best",
+				Payload: "mamma-mia-paylodia-belissima",
+			}
+
+			if test.ctx == nil {
+				test.ctx = context.Background()
+			}
+
+			err := c.AbortTransaction(test.ctx, u.Host, tx)
+			assert.NotNil(t, err)
+
+			if test.expectedErr != nil {
+				assert.Equal(t, test.expectedErr, err)
+			}
+
+			if test.expectedErrContains != "" {
+				assert.Contains(t, err.Error(), test.expectedErrContains)
+			}
+		})
+	}
+}
+
+func TestCommitTransaction(t *testing.T) {
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		w.WriteHeader(http.StatusNoContent)
+	}
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	u, _ := url.Parse(server.URL)
+
+	c := NewClusterSchema(&http.Client{})
+
+	tx := &cluster.Transaction{
+		ID:      "am-i-going-to-be-cancelled",
+		Type:    "the worst",
+		Payload: "",
+	}
+
+	err := c.CommitTransaction(context.Background(), u.Host, tx)
+	assert.Nil(t, err)
+}
+
+func TestCommitTransactionUnhappyPaths(t *testing.T) {
+	type test struct {
+		name                string
+		handler             http.HandlerFunc
+		expectedErr         error
+		expectedErrContains string
+		ctx                 context.Context
+		shutdownPrematurely bool
+	}
+
+	expiredCtx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tests := []test{
+		{
+			name: "arbitrary 500",
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				defer r.Body.Close()
+				w.WriteHeader(http.StatusInternalServerError)
+				w.Write([]byte("nope!"))
+			},
+			expectedErrContains: "nope!",
+		},
+		{
+			name: "expired ctx",
+			ctx:  expiredCtx,
+			handler: func(w http.ResponseWriter, r *http.Request) {
+			},
+			expectedErrContains: "context",
+		},
+		{
+			name:                "remote server shut down",
+			shutdownPrematurely: true,
+			handler: func(w http.ResponseWriter, r *http.Request) {
+			},
+			expectedErrContains: "refused",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(test.handler))
+			if test.shutdownPrematurely {
+				server.Close()
+			} else {
+				defer server.Close()
+			}
+
+			u, _ := url.Parse(server.URL)
+
+			c := NewClusterSchema(&http.Client{})
+
+			tx := &cluster.Transaction{
+				ID:      "12345",
+				Type:    "the best",
+				Payload: "mamma-mia-paylodia-belissima",
+			}
+
+			if test.ctx == nil {
+				test.ctx = context.Background()
+			}
+
+			err := c.CommitTransaction(test.ctx, u.Host, tx)
+			assert.NotNil(t, err)
+
+			if test.expectedErr != nil {
+				assert.Equal(t, test.expectedErr, err)
+			}
+
+			if test.expectedErrContains != "" {
+				assert.Contains(t, err.Error(), test.expectedErrContains)
+			}
+		})
+	}
+}

adapters/clients/remote_index.go

@@ -0,0 +1,839 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
+	"github.com/weaviate/weaviate/adapters/handlers/rest/clusterapi"
+	"github.com/weaviate/weaviate/entities/additional"
+	"github.com/weaviate/weaviate/entities/aggregation"
+	"github.com/weaviate/weaviate/entities/filters"
+	"github.com/weaviate/weaviate/entities/search"
+	"github.com/weaviate/weaviate/entities/searchparams"
+	"github.com/weaviate/weaviate/entities/storobj"
+	"github.com/weaviate/weaviate/usecases/objects"
+	"github.com/weaviate/weaviate/usecases/scaler"
+)
+
+type RemoteIndex struct {
+	retryClient
+}
+
+func NewRemoteIndex(httpClient *http.Client) *RemoteIndex {
+	return &RemoteIndex{retryClient: retryClient{
+		client:  httpClient,
+		retryer: newRetryer(),
+	}}
+}
+
+func (c *RemoteIndex) PutObject(ctx context.Context, hostName, indexName,
+	shardName string, obj *storobj.Object,
+) error {
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects", indexName, shardName)
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	marshalled, err := clusterapi.IndicesPayloads.SingleObject.Marshal(obj)
+	if err != nil {
+		return errors.Wrap(err, "marshal payload")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(marshalled))
+	if err != nil {
+		return errors.Wrap(err, "open http request")
+	}
+
+	clusterapi.IndicesPayloads.SingleObject.SetContentTypeHeaderReq(req)
+	res, err := c.client.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusNoContent {
+		body, _ := io.ReadAll(res.Body)
+		return errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	return nil
+}
+
+func duplicateErr(in error, count int) []error {
+	out := make([]error, count)
+	for i := range out {
+		out[i] = in
+	}
+	return out
+}
+
+func (c *RemoteIndex) BatchPutObjects(ctx context.Context, hostName, indexName,
+	shardName string, objs []*storobj.Object, _ *additional.ReplicationProperties,
+) []error {
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects", indexName, shardName)
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	marshalled, err := clusterapi.IndicesPayloads.ObjectList.Marshal(objs)
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "marshal payload"), len(objs))
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(marshalled))
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "open http request"), len(objs))
+	}
+
+	clusterapi.IndicesPayloads.ObjectList.SetContentTypeHeaderReq(req)
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "send http request"), len(objs))
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		return duplicateErr(errors.Errorf("unexpected status code %d (%s)",
+			res.StatusCode, body), len(objs))
+	}
+
+	if ct, ok := clusterapi.IndicesPayloads.ErrorList.
+		CheckContentTypeHeader(res); !ok {
+		return duplicateErr(errors.Errorf("unexpected content type: %s",
+			ct), len(objs))
+	}
+
+	resBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "ready body"), len(objs))
+	}
+
+	return clusterapi.IndicesPayloads.ErrorList.Unmarshal(resBytes)
+}
+
+func (c *RemoteIndex) BatchAddReferences(ctx context.Context, hostName, indexName,
+	shardName string, refs objects.BatchReferences,
+) []error {
+	path := fmt.Sprintf("/indices/%s/shards/%s/references", indexName, shardName)
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	marshalled, err := clusterapi.IndicesPayloads.ReferenceList.Marshal(refs)
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "marshal payload"), len(refs))
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(marshalled))
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "open http request"), len(refs))
+	}
+
+	clusterapi.IndicesPayloads.ReferenceList.SetContentTypeHeaderReq(req)
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "send http request"), len(refs))
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		return duplicateErr(errors.Errorf("unexpected status code %d (%s)",
+			res.StatusCode, body), len(refs))
+	}
+
+	if ct, ok := clusterapi.IndicesPayloads.ErrorList.
+		CheckContentTypeHeader(res); !ok {
+		return duplicateErr(errors.Errorf("unexpected content type: %s",
+			ct), len(refs))
+	}
+
+	resBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return duplicateErr(errors.Wrap(err, "ready body"), len(refs))
+	}
+
+	return clusterapi.IndicesPayloads.ErrorList.Unmarshal(resBytes)
+}
+
+func (c *RemoteIndex) GetObject(ctx context.Context, hostName, indexName,
+	shardName string, id strfmt.UUID, selectProps search.SelectProperties,
+	additional additional.Properties,
+) (*storobj.Object, error) {
+	selectPropsBytes, err := json.Marshal(selectProps)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal selectProps props")
+	}
+
+	additionalBytes, err := json.Marshal(additional)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal additional props")
+	}
+
+	selectPropsEncoded := base64.StdEncoding.EncodeToString(selectPropsBytes)
+	additionalEncoded := base64.StdEncoding.EncodeToString(additionalBytes)
+
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects/%s", indexName, shardName, id)
+	method := http.MethodGet
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+	q := url.Query()
+	q.Set("additional", additionalEncoded)
+	q.Set("selectProperties", selectPropsEncoded)
+	url.RawQuery = q.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "open http request")
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		// this is a legitimate case - the requested ID doesn't exist, don't try
+		// to unmarshal anything
+		return nil, nil
+	}
+
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		return nil, errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	ct, ok := clusterapi.IndicesPayloads.SingleObject.CheckContentTypeHeader(res)
+	if !ok {
+		return nil, errors.Errorf("unknown content type %s", ct)
+	}
+
+	objBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "read body")
+	}
+
+	obj, err := clusterapi.IndicesPayloads.SingleObject.Unmarshal(objBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshal body")
+	}
+
+	return obj, nil
+}
+
+func (c *RemoteIndex) Exists(ctx context.Context, hostName, indexName,
+	shardName string, id strfmt.UUID,
+) (bool, error) {
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects/%s", indexName, shardName, id)
+	method := http.MethodGet
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+	q := url.Query()
+	q.Set("check_exists", "true")
+	url.RawQuery = q.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return false, errors.Wrap(err, "open http request")
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return false, errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		// this is a legitimate case - the requested ID doesn't exist, don't try
+		// to unmarshal anything
+		return false, nil
+	}
+
+	if res.StatusCode != http.StatusNoContent {
+		body, _ := io.ReadAll(res.Body)
+		return false, errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	return true, nil
+}
+
+func (c *RemoteIndex) DeleteObject(ctx context.Context, hostName, indexName,
+	shardName string, id strfmt.UUID,
+) error {
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects/%s", indexName, shardName, id)
+	method := http.MethodDelete
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return errors.Wrap(err, "open http request")
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		// this is a legitimate case - the requested ID doesn't exist, don't try
+		// to unmarshal anything, we can assume it was already deleted
+		return nil
+	}
+
+	if res.StatusCode != http.StatusNoContent {
+		body, _ := io.ReadAll(res.Body)
+		return errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	return nil
+}
+
+func (c *RemoteIndex) MergeObject(ctx context.Context, hostName, indexName,
+	shardName string, mergeDoc objects.MergeDocument,
+) error {
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects/%s", indexName, shardName,
+		mergeDoc.ID)
+	method := http.MethodPatch
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	marshalled, err := clusterapi.IndicesPayloads.MergeDoc.Marshal(mergeDoc)
+	if err != nil {
+		return errors.Wrap(err, "marshal payload")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(marshalled))
+	if err != nil {
+		return errors.Wrap(err, "open http request")
+	}
+
+	clusterapi.IndicesPayloads.MergeDoc.SetContentTypeHeaderReq(req)
+	res, err := c.client.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusNoContent {
+		body, _ := io.ReadAll(res.Body)
+		return errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	return nil
+}
+
+func (c *RemoteIndex) MultiGetObjects(ctx context.Context, hostName, indexName,
+	shardName string, ids []strfmt.UUID,
+) ([]*storobj.Object, error) {
+	idsBytes, err := json.Marshal(ids)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal selectProps props")
+	}
+
+	idsEncoded := base64.StdEncoding.EncodeToString(idsBytes)
+
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects", indexName, shardName)
+	method := http.MethodGet
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+	q := url.Query()
+	q.Set("ids", idsEncoded)
+	url.RawQuery = q.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "open http request")
+	}
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		// this is a legitimate case - the requested ID doesn't exist, don't try
+		// to unmarshal anything
+		return nil, nil
+	}
+
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		return nil, errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	ct, ok := clusterapi.IndicesPayloads.ObjectList.CheckContentTypeHeader(res)
+	if !ok {
+		return nil, errors.Errorf("unexpected content type: %s", ct)
+	}
+
+	bodyBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "read response body")
+	}
+
+	objs, err := clusterapi.IndicesPayloads.ObjectList.Unmarshal(bodyBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshal objects")
+	}
+
+	return objs, nil
+}
+
+func (c *RemoteIndex) SearchShard(ctx context.Context, host, index, shard string,
+	vector []float32, limit int,
+	filters *filters.LocalFilter,
+	keywordRanking *searchparams.KeywordRanking,
+	sort []filters.Sort,
+	cursor *filters.Cursor,
+	groupBy *searchparams.GroupBy,
+	additional additional.Properties,
+) ([]*storobj.Object, []float32, error) {
+	// new request
+	body, err := clusterapi.IndicesPayloads.SearchParams.
+		Marshal(vector, limit, filters, keywordRanking, sort, cursor, groupBy, additional)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal request payload: %w", err)
+	}
+	url := url.URL{
+		Scheme: "http",
+		Host:   host,
+		Path:   fmt.Sprintf("/indices/%s/shards/%s/objects/_search", index, shard),
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url.String(), bytes.NewReader(body))
+	if err != nil {
+		return nil, nil, fmt.Errorf("create http request: %w", err)
+	}
+	clusterapi.IndicesPayloads.SearchParams.SetContentTypeHeaderReq(req)
+
+	// send request
+	resp := &searchShardResp{}
+	err = c.doWithCustomMarshaller(c.timeoutUnit*20, req, body, resp.decode)
+	return resp.Objects, resp.Distributions, err
+}
+
+type searchShardResp struct {
+	Objects       []*storobj.Object
+	Distributions []float32
+}
+
+func (r *searchShardResp) decode(data []byte) (err error) {
+	r.Objects, r.Distributions, err = clusterapi.IndicesPayloads.SearchResults.Unmarshal(data)
+	return
+}
+
+type aggregateResp struct {
+	Result *aggregation.Result
+}
+
+func (r *aggregateResp) decode(data []byte) (err error) {
+	r.Result, err = clusterapi.IndicesPayloads.AggregationResult.Unmarshal(data)
+	return
+}
+
+func (c *RemoteIndex) Aggregate(ctx context.Context, hostName, index,
+	shard string, params aggregation.Params,
+) (*aggregation.Result, error) {
+	// create new request
+	body, err := clusterapi.IndicesPayloads.AggregationParams.Marshal(params)
+	if err != nil {
+		return nil, fmt.Errorf("marshal request payload: %w", err)
+	}
+
+	url := &url.URL{
+		Scheme: "http",
+		Host:   hostName,
+		Path:   fmt.Sprintf("/indices/%s/shards/%s/objects/_aggregations", index, shard),
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url.String(), bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("create http request: %w", err)
+	}
+	clusterapi.IndicesPayloads.AggregationParams.SetContentTypeHeaderReq(req)
+
+	// send request
+	resp := &aggregateResp{}
+	err = c.doWithCustomMarshaller(c.timeoutUnit*20, req, body, resp.decode)
+	return resp.Result, err
+}
+
+func (c *RemoteIndex) FindUUIDs(ctx context.Context, hostName, indexName,
+	shardName string, filters *filters.LocalFilter,
+) ([]strfmt.UUID, error) {
+	paramsBytes, err := clusterapi.IndicesPayloads.FindUUIDsParams.Marshal(filters)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal request payload")
+	}
+
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects/_find", indexName, shardName)
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(paramsBytes))
+	if err != nil {
+		return nil, errors.Wrap(err, "open http request")
+	}
+
+	clusterapi.IndicesPayloads.FindUUIDsParams.SetContentTypeHeaderReq(req)
+	res, err := c.client.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "send http request")
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		return nil, errors.Errorf("unexpected status code %d (%s)", res.StatusCode,
+			body)
+	}
+
+	resBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "read body")
+	}
+
+	ct, ok := clusterapi.IndicesPayloads.FindUUIDsResults.CheckContentTypeHeader(res)
+	if !ok {
+		return nil, errors.Errorf("unexpected content type: %s", ct)
+	}
+
+	uuids, err := clusterapi.IndicesPayloads.FindUUIDsResults.Unmarshal(resBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshal body")
+	}
+	return uuids, nil
+}
+
+func (c *RemoteIndex) DeleteObjectBatch(ctx context.Context, hostName, indexName, shardName string,
+	uuids []strfmt.UUID, dryRun bool,
+) objects.BatchSimpleObjects {
+	path := fmt.Sprintf("/indices/%s/shards/%s/objects", indexName, shardName)
+	method := http.MethodDelete
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	marshalled, err := clusterapi.IndicesPayloads.BatchDeleteParams.Marshal(uuids, dryRun)
+	if err != nil {
+		err := errors.Wrap(err, "marshal payload")
+		return objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: err}}
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(),
+		bytes.NewReader(marshalled))
+	if err != nil {
+		err := errors.Wrap(err, "open http request")
+		return objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: err}}
+	}
+
+	clusterapi.IndicesPayloads.BatchDeleteParams.SetContentTypeHeaderReq(req)
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		err := errors.Wrap(err, "send http request")
+		return objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: err}}
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		err := errors.Errorf("unexpected status code %d (%s)", res.StatusCode, body)
+		return objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: err}}
+	}
+
+	if ct, ok := clusterapi.IndicesPayloads.BatchDeleteResults.
+		CheckContentTypeHeader(res); !ok {
+		err := errors.Errorf("unexpected content type: %s", ct)
+		return objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: err}}
+	}
+
+	resBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		err := errors.Wrap(err, "ready body")
+		return objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: err}}
+	}
+
+	batchDeleteResults, err := clusterapi.IndicesPayloads.BatchDeleteResults.Unmarshal(resBytes)
+	if err != nil {
+		err := errors.Wrap(err, "unmarshal body")
+		return objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: err}}
+	}
+
+	return batchDeleteResults
+}
+
+func (c *RemoteIndex) GetShardQueueSize(ctx context.Context,
+	hostName, indexName, shardName string,
+) (int64, error) {
+	path := fmt.Sprintf("/indices/%s/shards/%s/queuesize", indexName, shardName)
+	method := http.MethodGet
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return 0, errors.Wrap(err, "open http request")
+	}
+	var size int64
+	clusterapi.IndicesPayloads.GetShardQueueSizeParams.SetContentTypeHeaderReq(req)
+	try := func(ctx context.Context) (bool, error) {
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusOK {
+			body, _ := io.ReadAll(res.Body)
+			return shouldRetry(code), fmt.Errorf("status code: %v body: (%s)", code, body)
+		}
+		resBytes, err := io.ReadAll(res.Body)
+		if err != nil {
+			return false, errors.Wrap(err, "read body")
+		}
+
+		ct, ok := clusterapi.IndicesPayloads.GetShardQueueSizeResults.CheckContentTypeHeader(res)
+		if !ok {
+			return false, errors.Errorf("unexpected content type: %s", ct)
+		}
+
+		size, err = clusterapi.IndicesPayloads.GetShardQueueSizeResults.Unmarshal(resBytes)
+		if err != nil {
+			return false, errors.Wrap(err, "unmarshal body")
+		}
+		return false, nil
+	}
+	return size, c.retry(ctx, 9, try)
+}
+
+func (c *RemoteIndex) GetShardStatus(ctx context.Context,
+	hostName, indexName, shardName string,
+) (string, error) {
+	path := fmt.Sprintf("/indices/%s/shards/%s/status", indexName, shardName)
+	method := http.MethodGet
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return "", errors.Wrap(err, "open http request")
+	}
+	var status string
+	clusterapi.IndicesPayloads.GetShardStatusParams.SetContentTypeHeaderReq(req)
+	try := func(ctx context.Context) (bool, error) {
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusOK {
+			body, _ := io.ReadAll(res.Body)
+			return shouldRetry(code), fmt.Errorf("status code: %v body: (%s)", code, body)
+		}
+		resBytes, err := io.ReadAll(res.Body)
+		if err != nil {
+			return false, errors.Wrap(err, "read body")
+		}
+
+		ct, ok := clusterapi.IndicesPayloads.GetShardStatusResults.CheckContentTypeHeader(res)
+		if !ok {
+			return false, errors.Errorf("unexpected content type: %s", ct)
+		}
+
+		status, err = clusterapi.IndicesPayloads.GetShardStatusResults.Unmarshal(resBytes)
+		if err != nil {
+			return false, errors.Wrap(err, "unmarshal body")
+		}
+		return false, nil
+	}
+	return status, c.retry(ctx, 9, try)
+}
+
+func (c *RemoteIndex) UpdateShardStatus(ctx context.Context, hostName, indexName, shardName,
+	targetStatus string,
+) error {
+	paramsBytes, err := clusterapi.IndicesPayloads.UpdateShardStatusParams.Marshal(targetStatus)
+	if err != nil {
+		return errors.Wrap(err, "marshal request payload")
+	}
+	path := fmt.Sprintf("/indices/%s/shards/%s/status", indexName, shardName)
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	try := func(ctx context.Context) (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, method, url.String(),
+			bytes.NewReader(paramsBytes))
+		if err != nil {
+			return false, fmt.Errorf("create http request: %w", err)
+		}
+		clusterapi.IndicesPayloads.UpdateShardStatusParams.SetContentTypeHeaderReq(req)
+
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusOK {
+			body, _ := io.ReadAll(res.Body)
+			return shouldRetry(code), fmt.Errorf("status code: %v body: (%s)", code, body)
+		}
+
+		return false, nil
+	}
+
+	return c.retry(ctx, 9, try)
+}
+
+func (c *RemoteIndex) PutFile(ctx context.Context, hostName, indexName,
+	shardName, fileName string, payload io.ReadSeekCloser,
+) error {
+	defer payload.Close()
+	path := fmt.Sprintf("/indices/%s/shards/%s/files/%s",
+		indexName, shardName, fileName)
+
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+	try := func(ctx context.Context) (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, method, url.String(), payload)
+		if err != nil {
+			return false, fmt.Errorf("create http request: %w", err)
+		}
+		clusterapi.IndicesPayloads.ShardFiles.SetContentTypeHeaderReq(req)
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusNoContent {
+			shouldRetry := shouldRetry(code)
+			if shouldRetry {
+				_, err := payload.Seek(0, 0)
+				shouldRetry = (err == nil)
+			}
+			body, _ := io.ReadAll(res.Body)
+			return shouldRetry, fmt.Errorf("status code: %v body: (%s)", code, body)
+		}
+		return false, nil
+	}
+
+	return c.retry(ctx, 12, try)
+}
+
+func (c *RemoteIndex) CreateShard(ctx context.Context,
+	hostName, indexName, shardName string,
+) error {
+	path := fmt.Sprintf("/indices/%s/shards/%s", indexName, shardName)
+
+	method := http.MethodPost
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return fmt.Errorf("create http request: %w", err)
+	}
+	try := func(ctx context.Context) (bool, error) {
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusCreated {
+			body, _ := io.ReadAll(res.Body)
+			return shouldRetry(code), fmt.Errorf("status code: %v body: (%s)", code, body)
+		}
+		return false, nil
+	}
+
+	return c.retry(ctx, 9, try)
+}
+
+func (c *RemoteIndex) ReInitShard(ctx context.Context,
+	hostName, indexName, shardName string,
+) error {
+	path := fmt.Sprintf("/indices/%s/shards/%s:reinit", indexName, shardName)
+
+	method := http.MethodPut
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	req, err := http.NewRequestWithContext(ctx, method, url.String(), nil)
+	if err != nil {
+		return fmt.Errorf("create http request: %w", err)
+	}
+	try := func(ctx context.Context) (bool, error) {
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusNoContent {
+			body, _ := io.ReadAll(res.Body)
+			return shouldRetry(code), fmt.Errorf("status code: %v body: (%s)", code, body)
+
+		}
+		return false, nil
+	}
+
+	return c.retry(ctx, 9, try)
+}
+
+func (c *RemoteIndex) IncreaseReplicationFactor(ctx context.Context,
+	hostName, indexName string, dist scaler.ShardDist,
+) error {
+	path := fmt.Sprintf("/replicas/indices/%s/replication-factor:increase", indexName)
+
+	method := http.MethodPut
+	url := url.URL{Scheme: "http", Host: hostName, Path: path}
+
+	body, err := clusterapi.IndicesPayloads.IncreaseReplicationFactor.Marshall(dist)
+	if err != nil {
+		return err
+	}
+	try := func(ctx context.Context) (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, method, url.String(), bytes.NewReader(body))
+		if err != nil {
+			return false, fmt.Errorf("create http request: %w", err)
+		}
+
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusNoContent {
+			body, _ := io.ReadAll(res.Body)
+			return shouldRetry(code), fmt.Errorf("status code: %v body: (%s)", code, body)
+		}
+		return false, nil
+	}
+	return c.retry(ctx, 34, try)
+}

adapters/clients/remote_index_test.go

@@ -0,0 +1,312 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+//	_       _
+//
+// __      _____  __ ___   ___  __ _| |_ ___
+//
+//	\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//	 \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//	  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//	 Copyright © 2016 - 2022 SeMI Technologies B.V. All rights reserved.
+//
+//	 CONTACT: [email protected]
+package clients
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/weaviate/weaviate/adapters/handlers/rest/clusterapi"
+)
+
+func TestRemoteIndexIncreaseRF(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	path := "/replicas/indices/C1/replication-factor:increase"
+	fs := newFakeRemoteIndexServer(t, http.MethodPut, path)
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newRemoteIndex(ts.Client())
+	t.Run("ConnectionError", func(t *testing.T) {
+		err := client.IncreaseReplicationFactor(ctx, "", "C1", nil)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+	n := 0
+	fs.doAfter = func(w http.ResponseWriter, r *http.Request) {
+		if n == 0 {
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if n == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+		} else {
+			w.WriteHeader(http.StatusNoContent)
+		}
+		n++
+	}
+	t.Run("Success", func(t *testing.T) {
+		err := client.IncreaseReplicationFactor(ctx, fs.host, "C1", nil)
+		assert.Nil(t, err)
+	})
+}
+
+func TestRemoteIndexReInitShardIn(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	path := "/indices/C1/shards/S1:reinit"
+	fs := newFakeRemoteIndexServer(t, http.MethodPut, path)
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newRemoteIndex(ts.Client())
+	t.Run("ConnectionError", func(t *testing.T) {
+		err := client.ReInitShard(ctx, "", "C1", "S1")
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+	n := 0
+	fs.doAfter = func(w http.ResponseWriter, r *http.Request) {
+		if n == 0 {
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if n == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+		} else {
+			w.WriteHeader(http.StatusNoContent)
+		}
+		n++
+	}
+	t.Run("Success", func(t *testing.T) {
+		err := client.ReInitShard(ctx, fs.host, "C1", "S1")
+		assert.Nil(t, err)
+	})
+}
+
+func TestRemoteIndexCreateShard(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	path := "/indices/C1/shards/S1"
+	fs := newFakeRemoteIndexServer(t, http.MethodPost, path)
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newRemoteIndex(ts.Client())
+	t.Run("ConnectionError", func(t *testing.T) {
+		err := client.CreateShard(ctx, "", "C1", "S1")
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+	n := 0
+	fs.doAfter = func(w http.ResponseWriter, r *http.Request) {
+		if n == 0 {
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if n == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+		} else {
+			w.WriteHeader(http.StatusCreated)
+		}
+		n++
+	}
+	t.Run("Success", func(t *testing.T) {
+		err := client.CreateShard(ctx, fs.host, "C1", "S1")
+		assert.Nil(t, err)
+	})
+}
+
+func TestRemoteIndexUpdateShardStatus(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	path := "/indices/C1/shards/S1/status"
+	fs := newFakeRemoteIndexServer(t, http.MethodPost, path)
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newRemoteIndex(ts.Client())
+	t.Run("ConnectionError", func(t *testing.T) {
+		err := client.UpdateShardStatus(ctx, "", "C1", "S1", "NewStatus")
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+	n := 0
+	fs.doAfter = func(w http.ResponseWriter, r *http.Request) {
+		if n == 0 {
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if n == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+		}
+		n++
+	}
+	t.Run("Success", func(t *testing.T) {
+		err := client.UpdateShardStatus(ctx, fs.host, "C1", "S1", "NewStatus")
+		assert.Nil(t, err)
+	})
+}
+
+func TestRemoteIndexShardStatus(t *testing.T) {
+	t.Parallel()
+	var (
+		ctx    = context.Background()
+		path   = "/indices/C1/shards/S1/status"
+		fs     = newFakeRemoteIndexServer(t, http.MethodGet, path)
+		Status = "READONLY"
+	)
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newRemoteIndex(ts.Client())
+	t.Run("ConnectionError", func(t *testing.T) {
+		_, err := client.GetShardStatus(ctx, "", "C1", "S1")
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+	n := 0
+	fs.doAfter = func(w http.ResponseWriter, r *http.Request) {
+		if n == 0 {
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if n == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+		} else if n == 2 {
+			w.Header().Set("content-type", "any")
+		} else if n == 3 {
+			clusterapi.IndicesPayloads.GetShardStatusResults.SetContentTypeHeader(w)
+		} else {
+			clusterapi.IndicesPayloads.GetShardStatusResults.SetContentTypeHeader(w)
+			bytes, _ := clusterapi.IndicesPayloads.GetShardStatusResults.Marshal(Status)
+			w.Write(bytes)
+		}
+		n++
+	}
+
+	t.Run("ContentType", func(t *testing.T) {
+		_, err := client.GetShardStatus(ctx, fs.host, "C1", "S1")
+		assert.NotNil(t, err)
+	})
+	t.Run("Status", func(t *testing.T) {
+		_, err := client.GetShardStatus(ctx, fs.host, "C1", "S1")
+		assert.NotNil(t, err)
+	})
+	t.Run("Success", func(t *testing.T) {
+		st, err := client.GetShardStatus(ctx, fs.host, "C1", "S1")
+		assert.Nil(t, err)
+		assert.Equal(t, "READONLY", st)
+	})
+}
+
+func TestRemoteIndexPutFile(t *testing.T) {
+	t.Parallel()
+	var (
+		ctx  = context.Background()
+		path = "/indices/C1/shards/S1/files/file1"
+		fs   = newFakeRemoteIndexServer(t, http.MethodPost, path)
+	)
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newRemoteIndex(ts.Client())
+
+	rsc := struct {
+		*strings.Reader
+		io.Closer
+	}{
+		strings.NewReader("hello, world"),
+		io.NopCloser(nil),
+	}
+	t.Run("ConnectionError", func(t *testing.T) {
+		err := client.PutFile(ctx, "", "C1", "S1", "file1", rsc)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+	n := 0
+	fs.doAfter = func(w http.ResponseWriter, r *http.Request) {
+		if n == 0 {
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if n == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+		} else {
+			w.WriteHeader(http.StatusNoContent)
+		}
+		n++
+	}
+
+	t.Run("Success", func(t *testing.T) {
+		err := client.PutFile(ctx, fs.host, "C1", "S1", "file1", rsc)
+		assert.Nil(t, err)
+	})
+}
+
+func newRemoteIndex(httpClient *http.Client) *RemoteIndex {
+	ri := NewRemoteIndex(httpClient)
+	ri.minBackOff = time.Millisecond * 1
+	ri.maxBackOff = time.Millisecond * 10
+	ri.timeoutUnit = time.Millisecond * 20
+	return ri
+}
+
+type fakeRemoteIndexServer struct {
+	method   string
+	path     string
+	host     string
+	doBefore func(w http.ResponseWriter, r *http.Request) error
+	doAfter  func(w http.ResponseWriter, r *http.Request)
+}
+
+func newFakeRemoteIndexServer(t *testing.T, method, path string) *fakeRemoteIndexServer {
+	f := &fakeRemoteIndexServer{
+		method: method,
+		path:   path,
+	}
+	f.doBefore = func(w http.ResponseWriter, r *http.Request) error {
+		if r.Method != f.method {
+			w.WriteHeader(http.StatusBadRequest)
+			return fmt.Errorf("method want %s got %s", method, r.Method)
+		}
+		if f.path != r.URL.Path {
+			w.WriteHeader(http.StatusBadRequest)
+			return fmt.Errorf("path want %s got %s", path, r.URL.Path)
+		}
+		return nil
+	}
+	return f
+}
+
+func (f *fakeRemoteIndexServer) server(t *testing.T) *httptest.Server {
+	if f.doBefore == nil {
+		f.doBefore = func(w http.ResponseWriter, r *http.Request) error {
+			if r.Method != f.method {
+				w.WriteHeader(http.StatusBadRequest)
+				return fmt.Errorf("method want %s got %s", f.method, r.Method)
+			}
+			if f.path != r.URL.Path {
+				w.WriteHeader(http.StatusBadRequest)
+				return fmt.Errorf("path want %s got %s", f.path, r.URL.Path)
+			}
+			return nil
+		}
+	}
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		if err := f.doBefore(w, r); err != nil {
+			t.Error(err)
+			return
+		}
+		if f.doAfter != nil {
+			f.doAfter(w, r)
+		}
+	}
+	serv := httptest.NewServer(http.HandlerFunc(handler))
+	f.host = serv.URL[7:]
+	return serv
+}

adapters/clients/replication.go

@@ -0,0 +1,311 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/go-openapi/strfmt"
+	"github.com/weaviate/weaviate/adapters/handlers/rest/clusterapi"
+	"github.com/weaviate/weaviate/entities/additional"
+	"github.com/weaviate/weaviate/entities/search"
+	"github.com/weaviate/weaviate/entities/storobj"
+	"github.com/weaviate/weaviate/usecases/objects"
+	"github.com/weaviate/weaviate/usecases/replica"
+)
+
+// ReplicationClient is to coordinate operations among replicas
+
+type replicationClient retryClient
+
+func NewReplicationClient(httpClient *http.Client) replica.Client {
+	return &replicationClient{
+		client:  httpClient,
+		retryer: newRetryer(),
+	}
+}
+
+// FetchObject fetches one object it exits
+func (c *replicationClient) FetchObject(ctx context.Context, host, index,
+	shard string, id strfmt.UUID, selectProps search.SelectProperties,
+	additional additional.Properties,
+) (objects.Replica, error) {
+	resp := objects.Replica{}
+	req, err := newHttpReplicaRequest(ctx, http.MethodGet, host, index, shard, "", id.String(), nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+	err = c.doCustomUnmarshal(c.timeoutUnit*20, req, nil, resp.UnmarshalBinary)
+	return resp, err
+}
+
+func (c *replicationClient) DigestObjects(ctx context.Context,
+	host, index, shard string, ids []strfmt.UUID,
+) (result []replica.RepairResponse, err error) {
+	var resp []replica.RepairResponse
+	body, err := json.Marshal(ids)
+	if err != nil {
+		return nil, fmt.Errorf("marshal digest objects input: %w", err)
+	}
+	req, err := newHttpReplicaRequest(
+		ctx, http.MethodGet, host, index, shard,
+		"", "_digest", bytes.NewReader(body))
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+	err = c.do(c.timeoutUnit*20, req, body, &resp)
+	return resp, err
+}
+
+func (c *replicationClient) OverwriteObjects(ctx context.Context,
+	host, index, shard string, vobjects []*objects.VObject,
+) ([]replica.RepairResponse, error) {
+	var resp []replica.RepairResponse
+	body, err := clusterapi.IndicesPayloads.VersionedObjectList.Marshal(vobjects)
+	if err != nil {
+		return nil, fmt.Errorf("encode request: %w", err)
+	}
+	req, err := newHttpReplicaRequest(
+		ctx, http.MethodPut, host, index, shard,
+		"", "_overwrite", bytes.NewReader(body))
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+	err = c.do(c.timeoutUnit*90, req, body, &resp)
+	return resp, err
+}
+
+func (c *replicationClient) FetchObjects(ctx context.Context, host,
+	index, shard string, ids []strfmt.UUID,
+) ([]objects.Replica, error) {
+	resp := make(objects.Replicas, len(ids))
+	idsBytes, err := json.Marshal(ids)
+	if err != nil {
+		return nil, fmt.Errorf("marshal ids: %w", err)
+	}
+
+	idsEncoded := base64.StdEncoding.EncodeToString(idsBytes)
+
+	req, err := newHttpReplicaRequest(ctx, http.MethodGet, host, index, shard, "", "", nil)
+	if err != nil {
+		return nil, fmt.Errorf("create http request: %w", err)
+	}
+
+	req.URL.RawQuery = url.Values{"ids": []string{idsEncoded}}.Encode()
+	err = c.doCustomUnmarshal(c.timeoutUnit*90, req, nil, resp.UnmarshalBinary)
+	return resp, err
+}
+
+func (c *replicationClient) PutObject(ctx context.Context, host, index,
+	shard, requestID string, obj *storobj.Object,
+) (replica.SimpleResponse, error) {
+	var resp replica.SimpleResponse
+	body, err := clusterapi.IndicesPayloads.SingleObject.Marshal(obj)
+	if err != nil {
+		return resp, fmt.Errorf("encode request: %w", err)
+	}
+
+	req, err := newHttpReplicaRequest(ctx, http.MethodPost, host, index, shard, requestID, "", nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+
+	clusterapi.IndicesPayloads.SingleObject.SetContentTypeHeaderReq(req)
+	err = c.do(c.timeoutUnit*90, req, body, &resp)
+	return resp, err
+}
+
+func (c *replicationClient) DeleteObject(ctx context.Context, host, index,
+	shard, requestID string, uuid strfmt.UUID,
+) (replica.SimpleResponse, error) {
+	var resp replica.SimpleResponse
+	req, err := newHttpReplicaRequest(ctx, http.MethodDelete, host, index, shard, requestID, uuid.String(), nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+
+	err = c.do(c.timeoutUnit*90, req, nil, &resp)
+	return resp, err
+}
+
+func (c *replicationClient) PutObjects(ctx context.Context, host, index,
+	shard, requestID string, objects []*storobj.Object,
+) (replica.SimpleResponse, error) {
+	var resp replica.SimpleResponse
+	body, err := clusterapi.IndicesPayloads.ObjectList.Marshal(objects)
+	if err != nil {
+		return resp, fmt.Errorf("encode request: %w", err)
+	}
+	req, err := newHttpReplicaRequest(ctx, http.MethodPost, host, index, shard, requestID, "", nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+
+	clusterapi.IndicesPayloads.ObjectList.SetContentTypeHeaderReq(req)
+	err = c.do(c.timeoutUnit*90, req, body, &resp)
+	return resp, err
+}
+
+func (c *replicationClient) MergeObject(ctx context.Context, host, index, shard, requestID string,
+	doc *objects.MergeDocument,
+) (replica.SimpleResponse, error) {
+	var resp replica.SimpleResponse
+	body, err := clusterapi.IndicesPayloads.MergeDoc.Marshal(*doc)
+	if err != nil {
+		return resp, fmt.Errorf("encode request: %w", err)
+	}
+
+	req, err := newHttpReplicaRequest(ctx, http.MethodPatch, host, index, shard,
+		requestID, doc.ID.String(), nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+
+	clusterapi.IndicesPayloads.MergeDoc.SetContentTypeHeaderReq(req)
+	err = c.do(c.timeoutUnit*90, req, body, &resp)
+	return resp, err
+}
+
+func (c *replicationClient) AddReferences(ctx context.Context, host, index,
+	shard, requestID string, refs []objects.BatchReference,
+) (replica.SimpleResponse, error) {
+	var resp replica.SimpleResponse
+	body, err := clusterapi.IndicesPayloads.ReferenceList.Marshal(refs)
+	if err != nil {
+		return resp, fmt.Errorf("encode request: %w", err)
+	}
+	req, err := newHttpReplicaRequest(ctx, http.MethodPost, host, index, shard,
+		requestID, "references", nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+
+	clusterapi.IndicesPayloads.ReferenceList.SetContentTypeHeaderReq(req)
+	err = c.do(c.timeoutUnit*90, req, body, &resp)
+	return resp, err
+}
+
+func (c *replicationClient) DeleteObjects(ctx context.Context, host, index, shard, requestID string,
+	uuids []strfmt.UUID, dryRun bool,
+) (resp replica.SimpleResponse, err error) {
+	body, err := clusterapi.IndicesPayloads.BatchDeleteParams.Marshal(uuids, dryRun)
+	if err != nil {
+		return resp, fmt.Errorf("encode request: %w", err)
+	}
+	req, err := newHttpReplicaRequest(ctx, http.MethodDelete, host, index, shard, requestID, "", nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+
+	clusterapi.IndicesPayloads.BatchDeleteParams.SetContentTypeHeaderReq(req)
+	err = c.do(c.timeoutUnit*90, req, body, &resp)
+	return resp, err
+}
+
+// Commit asks a host to commit and stores the response in the value pointed to by resp
+func (c *replicationClient) Commit(ctx context.Context, host, index, shard string, requestID string, resp interface{}) error {
+	req, err := newHttpReplicaCMD(host, "commit", index, shard, requestID, nil)
+	if err != nil {
+		return fmt.Errorf("create http request: %w", err)
+	}
+
+	return c.do(c.timeoutUnit*90, req, nil, resp)
+}
+
+func (c *replicationClient) Abort(ctx context.Context, host, index, shard, requestID string) (
+	resp replica.SimpleResponse, err error,
+) {
+	req, err := newHttpReplicaCMD(host, "abort", index, shard, requestID, nil)
+	if err != nil {
+		return resp, fmt.Errorf("create http request: %w", err)
+	}
+
+	err = c.do(c.timeoutUnit*5, req, nil, &resp)
+	return resp, err
+}
+
+func newHttpReplicaRequest(ctx context.Context, method, host, index, shard, requestId, suffix string, body io.Reader) (*http.Request, error) {
+	path := fmt.Sprintf("/replicas/indices/%s/shards/%s/objects", index, shard)
+	if suffix != "" {
+		path = fmt.Sprintf("%s/%s", path, suffix)
+	}
+	u := url.URL{
+		Scheme: "http",
+		Host:   host,
+		Path:   path,
+	}
+
+	if requestId != "" {
+		u.RawQuery = url.Values{replica.RequestKey: []string{requestId}}.Encode()
+	}
+
+	return http.NewRequestWithContext(ctx, method, u.String(), body)
+}
+
+func newHttpReplicaCMD(host, cmd, index, shard, requestId string, body io.Reader) (*http.Request, error) {
+	path := fmt.Sprintf("/replicas/indices/%s/shards/%s:%s", index, shard, cmd)
+	q := url.Values{replica.RequestKey: []string{requestId}}.Encode()
+	url := url.URL{Scheme: "http", Host: host, Path: path, RawQuery: q}
+	return http.NewRequest(http.MethodPost, url.String(), body)
+}
+
+func (c *replicationClient) do(timeout time.Duration, req *http.Request, body []byte, resp interface{}) (err error) {
+	ctx, cancel := context.WithTimeout(req.Context(), timeout)
+	defer cancel()
+	try := func(ctx context.Context) (bool, error) {
+		if body != nil {
+			req.Body = io.NopCloser(bytes.NewReader(body))
+		}
+		res, err := c.client.Do(req)
+		if err != nil {
+			return ctx.Err() == nil, fmt.Errorf("connect: %w", err)
+		}
+		defer res.Body.Close()
+
+		if code := res.StatusCode; code != http.StatusOK {
+			b, _ := io.ReadAll(res.Body)
+			return shouldRetry(code), fmt.Errorf("status code: %v, error: %s", code, b)
+		}
+		if err := json.NewDecoder(res.Body).Decode(resp); err != nil {
+			return false, fmt.Errorf("decode response: %w", err)
+		}
+		return false, nil
+	}
+	return c.retry(ctx, 9, try)
+}
+
+func (c *replicationClient) doCustomUnmarshal(timeout time.Duration,
+	req *http.Request, body []byte, decode func([]byte) error,
+) (err error) {
+	return (*retryClient)(c).doWithCustomMarshaller(timeout, req, body, decode)
+}
+
+// backOff return a new random duration in the interval [d, 3d].
+// It implements truncated exponential back-off with introduced jitter.
+func backOff(d time.Duration) time.Duration {
+	return time.Duration(float64(d.Nanoseconds()*2) * (0.5 + rand.Float64()))
+}
+
+func shouldRetry(code int) bool {
+	return code == http.StatusInternalServerError ||
+		code == http.StatusTooManyRequests ||
+		code == http.StatusServiceUnavailable
+}

adapters/clients/replication_test.go

@@ -0,0 +1,592 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package clients
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-openapi/strfmt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/weaviate/weaviate/entities/additional"
+	"github.com/weaviate/weaviate/entities/models"
+	"github.com/weaviate/weaviate/entities/storobj"
+	"github.com/weaviate/weaviate/usecases/objects"
+	"github.com/weaviate/weaviate/usecases/replica"
+)
+
+const (
+	RequestError             = "RIDNotFound"
+	RequestSuccess           = "RIDSuccess"
+	RequestInternalError     = "RIDInternal"
+	RequestMalFormedResponse = "RIDMalFormed"
+)
+
+const (
+	UUID1 = strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168241")
+	UUID2 = strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168242")
+)
+
+type fakeServer struct {
+	method         string
+	path           string
+	RequestError   replica.SimpleResponse
+	RequestSuccess replica.SimpleResponse
+	host           string
+}
+
+func newFakeReplicationServer(t *testing.T, method, path string) *fakeServer {
+	return &fakeServer{
+		method:         method,
+		path:           path,
+		RequestError:   replica.SimpleResponse{Errors: []replica.Error{{Msg: "error"}}},
+		RequestSuccess: replica.SimpleResponse{},
+	}
+}
+
+func (f *fakeServer) server(t *testing.T) *httptest.Server {
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != f.method {
+			t.Errorf("method want %s got %s", f.method, r.Method)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if f.path != r.URL.Path {
+			t.Errorf("path want %s got %s", f.path, r.URL.Path)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		requestID := r.URL.Query().Get(replica.RequestKey)
+		switch requestID {
+		case RequestInternalError:
+			w.WriteHeader(http.StatusInternalServerError)
+		case RequestError:
+			bytes, _ := json.Marshal(&f.RequestError)
+			w.Write(bytes)
+		case RequestSuccess:
+			bytes, _ := json.Marshal(&replica.SimpleResponse{})
+			w.Write(bytes)
+		case RequestMalFormedResponse:
+			w.Write([]byte(`mal formed`))
+		}
+	}
+	serv := httptest.NewServer(http.HandlerFunc(handler))
+	f.host = serv.URL[7:]
+	return serv
+}
+
+func anyObject(uuid strfmt.UUID) models.Object {
+	return models.Object{
+		Class:              "C1",
+		CreationTimeUnix:   900000000001,
+		LastUpdateTimeUnix: 900000000002,
+		ID:                 uuid,
+		Properties: map[string]interface{}{
+			"stringProp":    "string",
+			"textProp":      "text",
+			"datePropArray": []string{"1980-01-01T00:00:00+02:00"},
+		},
+	}
+}
+
+func TestReplicationPutObject(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	f := newFakeReplicationServer(t, http.MethodPost, "/replicas/indices/C1/shards/S1/objects")
+	ts := f.server(t)
+	defer ts.Close()
+
+	client := newReplicationClient(ts.Client())
+	t.Run("EncodeRequest", func(t *testing.T) {
+		obj := &storobj.Object{}
+		_, err := client.PutObject(ctx, "Node1", "C1", "S1", "RID", obj)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "encode")
+	})
+
+	obj := &storobj.Object{MarshallerVersion: 1, Object: anyObject(UUID1)}
+	t.Run("ConnectionError", func(t *testing.T) {
+		_, err := client.PutObject(ctx, "", "C1", "S1", "", obj)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		resp, err := client.PutObject(ctx, f.host, "C1", "S1", RequestError, obj)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: f.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		_, err := client.PutObject(ctx, f.host, "C1", "S1", RequestMalFormedResponse, obj)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+
+	t.Run("ServerInternalError", func(t *testing.T) {
+		_, err := client.PutObject(ctx, f.host, "C1", "S1", RequestInternalError, obj)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationDeleteObject(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	uuid := UUID1
+	path := "/replicas/indices/C1/shards/S1/objects/" + uuid.String()
+	fs := newFakeReplicationServer(t, http.MethodDelete, path)
+	ts := fs.server(t)
+	defer ts.Close()
+
+	client := newReplicationClient(ts.Client())
+	t.Run("ConnectionError", func(t *testing.T) {
+		_, err := client.DeleteObject(ctx, "", "C1", "S1", "", uuid)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		resp, err := client.DeleteObject(ctx, fs.host, "C1", "S1", RequestError, uuid)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: fs.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		_, err := client.DeleteObject(ctx, fs.host, "C1", "S1", RequestMalFormedResponse, uuid)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+
+	t.Run("ServerInternalError", func(t *testing.T) {
+		_, err := client.DeleteObject(ctx, fs.host, "C1", "S1", RequestInternalError, uuid)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationPutObjects(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	fs := newFakeReplicationServer(t, http.MethodPost, "/replicas/indices/C1/shards/S1/objects")
+	fs.RequestError.Errors = append(fs.RequestError.Errors, replica.Error{Msg: "error2"})
+	ts := fs.server(t)
+	defer ts.Close()
+
+	client := newReplicationClient(ts.Client())
+	t.Run("EncodeRequest", func(t *testing.T) {
+		objs := []*storobj.Object{{}}
+		_, err := client.PutObjects(ctx, "Node1", "C1", "S1", "RID", objs)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "encode")
+	})
+
+	objects := []*storobj.Object{
+		{MarshallerVersion: 1, Object: anyObject(UUID1)},
+		{MarshallerVersion: 1, Object: anyObject(UUID2)},
+	}
+
+	t.Run("ConnectionError", func(t *testing.T) {
+		_, err := client.PutObjects(ctx, "", "C1", "S1", "", objects)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		resp, err := client.PutObjects(ctx, fs.host, "C1", "S1", RequestError, objects)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: fs.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		_, err := client.PutObjects(ctx, fs.host, "C1", "S1", RequestMalFormedResponse, objects)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+
+	t.Run("ServerInternalError", func(t *testing.T) {
+		_, err := client.PutObjects(ctx, fs.host, "C1", "S1", RequestInternalError, objects)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationMergeObject(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	uuid := UUID1
+	f := newFakeReplicationServer(t, http.MethodPatch, "/replicas/indices/C1/shards/S1/objects/"+uuid.String())
+	ts := f.server(t)
+	defer ts.Close()
+
+	client := newReplicationClient(ts.Client())
+	doc := &objects.MergeDocument{ID: uuid}
+	t.Run("ConnectionError", func(t *testing.T) {
+		_, err := client.MergeObject(ctx, "", "C1", "S1", "", doc)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		resp, err := client.MergeObject(ctx, f.host, "C1", "S1", RequestError, doc)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: f.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		_, err := client.MergeObject(ctx, f.host, "C1", "S1", RequestMalFormedResponse, doc)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+
+	t.Run("ServerInternalError", func(t *testing.T) {
+		_, err := client.MergeObject(ctx, f.host, "C1", "S1", RequestInternalError, doc)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationAddReferences(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	fs := newFakeReplicationServer(t, http.MethodPost, "/replicas/indices/C1/shards/S1/objects/references")
+	fs.RequestError.Errors = append(fs.RequestError.Errors, replica.Error{Msg: "error2"})
+	ts := fs.server(t)
+	defer ts.Close()
+
+	client := newReplicationClient(ts.Client())
+	refs := []objects.BatchReference{{OriginalIndex: 1}, {OriginalIndex: 2}}
+	t.Run("ConnectionError", func(t *testing.T) {
+		_, err := client.AddReferences(ctx, "", "C1", "S1", "", refs)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		resp, err := client.AddReferences(ctx, fs.host, "C1", "S1", RequestError, refs)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: fs.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		_, err := client.AddReferences(ctx, fs.host, "C1", "S1", RequestMalFormedResponse, refs)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+
+	t.Run("ServerInternalError", func(t *testing.T) {
+		_, err := client.AddReferences(ctx, fs.host, "C1", "S1", RequestInternalError, refs)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationDeleteObjects(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	fs := newFakeReplicationServer(t, http.MethodDelete, "/replicas/indices/C1/shards/S1/objects")
+	fs.RequestError.Errors = append(fs.RequestError.Errors, replica.Error{Msg: "error2"})
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newReplicationClient(ts.Client())
+
+	uuids := []strfmt.UUID{strfmt.UUID("1"), strfmt.UUID("2")}
+	t.Run("ConnectionError", func(t *testing.T) {
+		_, err := client.DeleteObjects(ctx, "", "C1", "S1", "", uuids, false)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		resp, err := client.DeleteObjects(ctx, fs.host, "C1", "S1", RequestError, uuids, false)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: fs.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		_, err := client.DeleteObjects(ctx, fs.host, "C1", "S1", RequestMalFormedResponse, uuids, false)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+
+	t.Run("ServerInternalError", func(t *testing.T) {
+		_, err := client.DeleteObjects(ctx, fs.host, "C1", "S1", RequestInternalError, uuids, false)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationAbort(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	path := "/replicas/indices/C1/shards/S1:abort"
+	fs := newFakeReplicationServer(t, http.MethodPost, path)
+	ts := fs.server(t)
+	defer ts.Close()
+	client := newReplicationClient(ts.Client())
+
+	t.Run("ConnectionError", func(t *testing.T) {
+		client := newReplicationClient(ts.Client())
+		client.maxBackOff = client.timeoutUnit * 20
+		_, err := client.Abort(ctx, "", "C1", "S1", "")
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		resp, err := client.Abort(ctx, fs.host, "C1", "S1", RequestError)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: fs.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		_, err := client.Abort(ctx, fs.host, "C1", "S1", RequestMalFormedResponse)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+	client.timeoutUnit = client.maxBackOff * 3
+	t.Run("ServerInternalError", func(t *testing.T) {
+		_, err := client.Abort(ctx, fs.host, "C1", "S1", RequestInternalError)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationCommit(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	path := "/replicas/indices/C1/shards/S1:commit"
+	fs := newFakeReplicationServer(t, http.MethodPost, path)
+	ts := fs.server(t)
+	defer ts.Close()
+	resp := replica.SimpleResponse{}
+	client := newReplicationClient(ts.Client())
+
+	t.Run("ConnectionError", func(t *testing.T) {
+		err := client.Commit(ctx, "", "C1", "S1", "", &resp)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect")
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		err := client.Commit(ctx, fs.host, "C1", "S1", RequestError, &resp)
+		assert.Nil(t, err)
+		assert.Equal(t, replica.SimpleResponse{Errors: fs.RequestError.Errors}, resp)
+	})
+
+	t.Run("DecodeResponse", func(t *testing.T) {
+		err := client.Commit(ctx, fs.host, "C1", "S1", RequestMalFormedResponse, &resp)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "decode response")
+	})
+
+	t.Run("ServerInternalError", func(t *testing.T) {
+		err := client.Commit(ctx, fs.host, "C1", "S1", RequestInternalError, &resp)
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "status code")
+	})
+}
+
+func TestReplicationFetchObject(t *testing.T) {
+	t.Parallel()
+
+	expected := objects.Replica{
+		ID: UUID1,
+		Object: &storobj.Object{
+			MarshallerVersion: 1,
+			Object: models.Object{
+				ID: UUID1,
+				Properties: map[string]interface{}{
+					"stringProp": "abc",
+				},
+			},
+			Vector:    []float32{1, 2, 3, 4, 5},
+			VectorLen: 5,
+		},
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := expected.MarshalBinary()
+		w.Write(b)
+	}))
+
+	c := newReplicationClient(server.Client())
+	resp, err := c.FetchObject(context.Background(), server.URL[7:],
+		"C1", "S1", expected.ID, nil, additional.Properties{})
+	require.Nil(t, err)
+	assert.Equal(t, expected.ID, resp.ID)
+	assert.Equal(t, expected.Deleted, resp.Deleted)
+	assert.EqualValues(t, expected.Object, resp.Object)
+}
+
+func TestReplicationFetchObjects(t *testing.T) {
+	t.Parallel()
+	expected := objects.Replicas{
+		{
+			ID: UUID1,
+			Object: &storobj.Object{
+				MarshallerVersion: 1,
+				Object: models.Object{
+					ID: UUID1,
+					Properties: map[string]interface{}{
+						"stringProp": "abc",
+					},
+				},
+				Vector:    []float32{1, 2, 3, 4, 5},
+				VectorLen: 5,
+			},
+		},
+		{
+			ID: UUID2,
+			Object: &storobj.Object{
+				MarshallerVersion: 1,
+				Object: models.Object{
+					ID: UUID2,
+					Properties: map[string]interface{}{
+						"floatProp": float64(123),
+					},
+				},
+				Vector:    []float32{10, 20, 30, 40, 50},
+				VectorLen: 5,
+			},
+		},
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := expected.MarshalBinary()
+		w.Write(b)
+	}))
+
+	c := newReplicationClient(server.Client())
+	resp, err := c.FetchObjects(context.Background(), server.URL[7:], "C1", "S1", []strfmt.UUID{expected[0].ID})
+	require.Nil(t, err)
+	require.Len(t, resp, 2)
+	assert.Equal(t, expected[0].ID, resp[0].ID)
+	assert.Equal(t, expected[0].Deleted, resp[0].Deleted)
+	assert.EqualValues(t, expected[0].Object, resp[0].Object)
+	assert.Equal(t, expected[1].ID, resp[1].ID)
+	assert.Equal(t, expected[1].Deleted, resp[1].Deleted)
+	assert.EqualValues(t, expected[1].Object, resp[1].Object)
+}
+
+func TestReplicationDigestObjects(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	expected := []replica.RepairResponse{
+		{
+			ID:         UUID1.String(),
+			UpdateTime: now.UnixMilli(),
+			Version:    1,
+		},
+		{
+			ID:         UUID2.String(),
+			UpdateTime: now.UnixMilli(),
+			Version:    1,
+		},
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := json.Marshal(expected)
+		w.Write(b)
+	}))
+
+	c := newReplicationClient(server.Client())
+	resp, err := c.DigestObjects(context.Background(), server.URL[7:], "C1", "S1", []strfmt.UUID{
+		strfmt.UUID(expected[0].ID),
+		strfmt.UUID(expected[1].ID),
+	})
+	require.Nil(t, err)
+	require.Len(t, resp, 2)
+	assert.Equal(t, expected[0].ID, resp[0].ID)
+	assert.Equal(t, expected[0].Deleted, resp[0].Deleted)
+	assert.Equal(t, expected[0].UpdateTime, resp[0].UpdateTime)
+	assert.Equal(t, expected[0].Version, resp[0].Version)
+	assert.Equal(t, expected[1].ID, resp[1].ID)
+	assert.Equal(t, expected[1].Deleted, resp[1].Deleted)
+	assert.Equal(t, expected[1].UpdateTime, resp[1].UpdateTime)
+	assert.Equal(t, expected[1].Version, resp[1].Version)
+}
+
+func TestReplicationOverwriteObjects(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	input := []*objects.VObject{
+		{
+			LatestObject: &models.Object{
+				ID:                 UUID1,
+				Class:              "C1",
+				CreationTimeUnix:   now.UnixMilli(),
+				LastUpdateTimeUnix: now.Add(time.Hour).UnixMilli(),
+				Properties: map[string]interface{}{
+					"stringProp": "abc",
+				},
+				Vector: []float32{1, 2, 3, 4, 5},
+			},
+			StaleUpdateTime: now.UnixMilli(),
+			Version:         0,
+		},
+	}
+	expected := []replica.RepairResponse{
+		{
+			ID:         UUID1.String(),
+			Version:    1,
+			UpdateTime: now.Add(time.Hour).UnixMilli(),
+		},
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := json.Marshal(expected)
+		w.Write(b)
+	}))
+
+	c := newReplicationClient(server.Client())
+	resp, err := c.OverwriteObjects(context.Background(), server.URL[7:], "C1", "S1", input)
+	require.Nil(t, err)
+	require.Len(t, resp, 1)
+	assert.Equal(t, expected[0].ID, resp[0].ID)
+	assert.Equal(t, expected[0].Version, resp[0].Version)
+	assert.Equal(t, expected[0].UpdateTime, resp[0].UpdateTime)
+}
+
+func TestExpBackOff(t *testing.T) {
+	N := 200
+	av := time.Duration(0)
+	delay := time.Nanosecond * 20
+	for i := 0; i < N; i++ {
+		av += backOff(delay)
+	}
+	av /= time.Duration(N)
+	if av < time.Nanosecond*30 || av > time.Nanosecond*50 {
+		t.Errorf("average time got %v", av)
+	}
+}
+
+func newReplicationClient(httpClient *http.Client) *replicationClient {
+	c := NewReplicationClient(httpClient).(*replicationClient)
+	c.minBackOff = time.Millisecond * 1
+	c.maxBackOff = time.Millisecond * 8
+	c.timeoutUnit = time.Millisecond * 20
+	return c
+}

adapters/handlers/graphql/common/fetch/filter.go

@@ -0,0 +1,146 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package fetch
+
+import (
+	"fmt"
+
+	"github.com/tailor-inc/graphql"
+	"github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions"
+	"github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters"
+)
+
+// FilterBuilder can build where filters for both local and
+type FilterBuilder struct {
+	prefix string
+}
+
+// NewFilterBuilder with kind and prefix
+func NewFilterBuilder(prefix string) *FilterBuilder {
+	return &FilterBuilder{
+		prefix: prefix,
+	}
+}
+
+// Build a where filter ArgumentConfig
+func (b *FilterBuilder) Build() *graphql.ArgumentConfig {
+	return &graphql.ArgumentConfig{
+		Description: descriptions.FetchWhereFilterFields,
+		Type: graphql.NewNonNull(graphql.NewInputObject(
+			graphql.InputObjectConfig{
+				Name:        fmt.Sprintf("%sFetchObjectWhereInpObj", b.prefix),
+				Fields:      b.fields(),
+				Description: descriptions.FetchWhereFilterFieldsInpObj,
+			},
+		)),
+	}
+}
+
+func (b *FilterBuilder) fields() graphql.InputObjectConfigFieldMap {
+	return graphql.InputObjectConfigFieldMap{
+		"class": &graphql.InputObjectFieldConfig{
+			Type:        graphql.NewNonNull(b.class()),
+			Description: descriptions.WhereClass,
+		},
+		"properties": &graphql.InputObjectFieldConfig{
+			Type:        graphql.NewNonNull(graphql.NewList(b.properties())),
+			Description: descriptions.WhereProperties,
+		},
+		"first": &graphql.InputObjectFieldConfig{
+			Type:        graphql.Int,
+			Description: descriptions.First,
+		},
+	}
+}
+
+func (b *FilterBuilder) properties() *graphql.InputObject {
+	elements := common_filters.BuildNew(fmt.Sprintf("%sFetchObject", b.prefix))
+
+	// Remove path and operands fields as they are not required here
+	delete(elements, "path")
+	delete(elements, "operands")
+
+	// make operator required
+	elements["operator"].Type = graphql.NewNonNull(elements["operator"].Type)
+
+	elements["certainty"] = &graphql.InputObjectFieldConfig{
+		Type:        graphql.NewNonNull(graphql.Float),
+		Description: descriptions.WhereCertainty,
+	}
+	elements["name"] = &graphql.InputObjectFieldConfig{
+		Type:        graphql.NewNonNull(graphql.String),
+		Description: descriptions.WhereName,
+	}
+	elements["keywords"] = &graphql.InputObjectFieldConfig{
+		Type:        graphql.NewList(b.keywordInpObj(fmt.Sprintf("%sFetchObjectWhereProperties", b.prefix))),
+		Description: descriptions.WhereKeywords,
+	}
+
+	networkFetchWhereInpObjPropertiesObj := graphql.NewInputObject(
+		graphql.InputObjectConfig{
+			Name:        fmt.Sprintf("%sFetchObjectWhereInpObjProperties", b.prefix),
+			Fields:      elements,
+			Description: descriptions.WhereProperties,
+		},
+	)
+
+	return networkFetchWhereInpObjPropertiesObj
+}
+
+func (b *FilterBuilder) keywordInpObj(prefix string) *graphql.InputObject {
+	return graphql.NewInputObject(
+		graphql.InputObjectConfig{
+			Name: fmt.Sprintf("%sKeywordsInpObj", prefix),
+			Fields: graphql.InputObjectConfigFieldMap{
+				"value": &graphql.InputObjectFieldConfig{
+					Type:        graphql.String,
+					Description: descriptions.WhereKeywordsValue,
+				},
+				"weight": &graphql.InputObjectFieldConfig{
+					Type:        graphql.Float,
+					Description: descriptions.WhereKeywordsWeight,
+				},
+			},
+			Description: descriptions.WhereKeywordsInpObj,
+		},
+	)
+}
+
+func (b *FilterBuilder) class() *graphql.InputObject {
+	filterClassElements := graphql.InputObjectConfigFieldMap{
+		"name": &graphql.InputObjectFieldConfig{
+			Type:        graphql.String,
+			Description: descriptions.WhereName,
+		},
+		"certainty": &graphql.InputObjectFieldConfig{
+			Type:        graphql.Float,
+			Description: descriptions.WhereCertainty,
+		},
+		"keywords": &graphql.InputObjectFieldConfig{
+			Type:        graphql.NewList(b.keywordInpObj(fmt.Sprintf("%sFetchObjectWhereClass", b.prefix))),
+			Description: descriptions.WhereKeywords,
+		},
+		"first": &graphql.InputObjectFieldConfig{
+			Type:        graphql.Int,
+			Description: descriptions.First,
+		},
+	}
+
+	networkFetchWhereInpObjClassInpObj := graphql.NewInputObject(
+		graphql.InputObjectConfig{
+			Name:        fmt.Sprintf("%sFetchObjectWhereInpObjClassInpObj", b.prefix),
+			Fields:      filterClassElements,
+			Description: descriptions.WhereClass,
+		},
+	)
+	return networkFetchWhereInpObjClassInpObj
+}

adapters/handlers/graphql/common/json_number.go

@@ -0,0 +1,60 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/tailor-inc/graphql"
+	"github.com/weaviate/weaviate/entities/aggregation"
+)
+
+// JSONNumberResolver turns json.Number types into number types usable by graphQL
+func JSONNumberResolver(p graphql.ResolveParams) (interface{}, error) {
+	switch v := p.Source.(type) {
+	case map[string]interface{}:
+		field, ok := v[p.Info.FieldName]
+		if !ok {
+			return nil, nil
+		}
+
+		switch n := field.(type) {
+		case json.Number:
+			return n.Float64()
+		case int64:
+			return float64(n), nil
+		case int:
+			return float64(n), nil
+		case float64:
+			return n, nil
+		}
+
+		return nil, fmt.Errorf("unknown number type for %t", field)
+
+	case map[string]float64:
+		return v[p.Info.FieldName], nil
+
+	case aggregation.Text:
+		switch p.Info.FieldName {
+		// case "count":
+		// 	// TODO gh-974: Support Count in text aggregations
+		// 	return nil, nil
+
+		default:
+			return nil, fmt.Errorf("fieldName '%s' does not match text aggregation", p.Info.FieldName)
+		}
+
+	default:
+		return nil, fmt.Errorf("json number resolver: unusable type %T", p.Source)
+	}
+}

adapters/handlers/graphql/common/json_number_test.go

@@ -0,0 +1,79 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tailor-inc/graphql"
+)
+
+type testCase struct {
+	input          interface{}
+	expectedOutput float64
+}
+
+func TestJSONNumberResolver(t *testing.T) {
+	tests := []testCase{
+		{
+			input:          json.Number("10"),
+			expectedOutput: 10.0,
+		},
+		{
+			input:          int(10),
+			expectedOutput: 10.0,
+		},
+		{
+			input:          int64(10),
+			expectedOutput: 10.0,
+		},
+		{
+			input:          float64(10),
+			expectedOutput: 10.0,
+		},
+	}
+
+	for _, test := range tests {
+		name := fmt.Sprintf("%#v -> %#v", test.input, test.expectedOutput)
+		t.Run(name, func(t *testing.T) {
+			result, err := JSONNumberResolver(resolveParams(test.input))
+			assert.Nil(t, err, "should not error")
+			assert.Equal(t, test.expectedOutput, result)
+		})
+	}
+}
+
+func resolveParams(input interface{}) graphql.ResolveParams {
+	return graphql.ResolveParams{
+		Source: map[string]interface{}{
+			"myField": input,
+		},
+		Info: graphql.ResolveInfo{FieldName: "myField"},
+	}
+}
+
+func TestNumberFieldNotPresent(t *testing.T) {
+	// shouldn't return anything, but also not error. This can otherwise lead to
+	// odd behavior when no entries are present, yet we asked for int props and
+	// type, see https://github.com/weaviate/weaviate/issues/775
+	params := graphql.ResolveParams{
+		Source: map[string]interface{}{},
+		Info:   graphql.ResolveInfo{FieldName: "myField"},
+	}
+
+	result, err := JSONNumberResolver(params)
+	assert.Nil(t, err)
+	assert.Equal(t, nil, result)
+}

adapters/handlers/graphql/descriptions/aggregate.go

@@ -0,0 +1,83 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// AGGREGATE
+const (
+	AggregateProperty = "Aggregate this property"
+	AggregateObjects  = "Aggregate Objects on a local Weaviate"
+)
+
+const GroupBy = "Specify which properties to group by"
+
+const (
+	AggregatePropertyObject = "An object containing Aggregation information about this property"
+)
+
+const AggregateObjectsObj = "An object allowing Aggregation of %ss on a local Weaviate"
+
+const (
+	AggregateMean      = "Aggregate on the mean of numeric property values"
+	AggregateSum       = "Aggregate on the sum of numeric property values"
+	AggregateMedian    = "Aggregate on the median of numeric property values"
+	AggregateMode      = "Aggregate on the mode of numeric property values"
+	AggregateMin       = "Aggregate on the minimum of numeric property values"
+	AggregateMax       = "Aggregate on the maximum of numeric property values"
+	AggregateCount     = "Aggregate on the total amount of found property values"
+	AggregateGroupedBy = "Indicates the group of returned data"
+)
+
+const AggregateNumericObj = "An object containing the %s of numeric properties"
+
+const AggregateCountObj = "An object containing countable properties"
+
+const AggregateGroupedByObj = "An object containing the path and value of the grouped property"
+
+const (
+	AggregateGroupedByGroupedByPath  = "The path of the grouped property"
+	AggregateGroupedByGroupedByValue = "The value of the grouped property"
+)
+
+// NETWORK
+const NetworkAggregateWeaviateObj = "An object containing Get Objects fields for network Weaviate instance: "
+
+const NetworkAggregate = "Perform Aggregation of Objects"
+
+const (
+	NetworkAggregateObj            = "An object allowing Aggregation of Objects"
+	NetworkAggregatePropertyObject = "An object containing Aggregation information about this property"
+)
+
+const NetworkAggregateThingsActionsObj = "An object allowing Aggregation of %ss on a network Weaviate"
+
+const (
+	NetworkAggregateMean      = "Aggregate on the mean of numeric property values"
+	NetworkAggregateSum       = "Aggregate on the sum of numeric property values"
+	NetworkAggregateMedian    = "Aggregate on the median of numeric property values"
+	NetworkAggregateMode      = "Aggregate on the mode of numeric property values"
+	NetworkAggregateMin       = "Aggregate on the minimum of numeric property values"
+	NetworkAggregateMax       = "Aggregate on the maximum of numeric property values"
+	NetworkAggregateCount     = "Aggregate on the total amount of found property values"
+	NetworkAggregateGroupedBy = "Indicates the group of returned data"
+)
+
+const NetworkAggregateNumericObj = "An object containing the %s of numeric properties"
+
+const NetworkAggregateCountObj = "An object containing countable properties"
+
+const NetworkAggregateGroupedByObj = "An object containing the path and value of the grouped property"
+
+const (
+	NetworkAggregateGroupedByGroupedByPath  = "The path of the grouped property"
+	NetworkAggregateGroupedByGroupedByValue = "The value of the grouped property"
+)

adapters/handlers/graphql/descriptions/explore.go

@@ -0,0 +1,30 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+const (
+	LocalExplore         = "Explore Concepts on a local weaviate with vector-aided search"
+	LocalExploreConcepts = "Explore Concepts on a local weaviate with vector-aided serach through keyword-based search terms"
+	VectorMovement       = "Move your search term closer to or further away from another vector described by keywords"
+	Keywords             = "Keywords are a list of search terms. Array type, e.g. [\"keyword 1\", \"keyword 2\"]"
+	Network              = "Set to true, if the exploration should include remote peers"
+	Limit                = "Limit the results set (usually fewer results mean faster queries)"
+	Offset               = "Offset of the results set (usually fewer results mean faster queries)"
+	Certainty            = "Normalized Distance between the result item and the search vector. Normalized to be between 0 (identical vectors) and 1 (perfect opposite)."
+	Distance             = "The required degree of similarity between an object's characteristics and the provided filter values"
+	Vector               = "Target vector to be used in kNN search"
+	Force                = "The force to apply for a particular movements. Must be between 0 and 1 where 0 is equivalent to no movement and 1 is equivalent to largest movement possible"
+	ClassName            = "Name of the Class"
+	ID                   = "Concept identifier in the uuid format"
+	Beacon               = "Concept identifier in the beacon format, such as weaviate://<hostname>/<kind>/id"
+)

adapters/handlers/graphql/descriptions/fetch.go

@@ -0,0 +1,55 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// Local
+const (
+	LocalFetch    = "Fetch Beacons that are similar to a specified concept from the Objects subsets on a Weaviate network"
+	LocalFetchObj = "An object used to perform a Fuzzy Fetch to search for Objects and Actions similar to a specified concept on a Weaviate network"
+)
+
+const (
+	LocalFetchObjects = "Perform a Fuzzy Fetch to Fetch Beacons similar to a specified concept on a Weaviate network from the Objects subset"
+	LocalFetchFuzzy   = "Perform a Fuzzy Fetch to Fetch Beacons similar to a specified concept on a Weaviate network from both the Objects subsets"
+)
+
+const (
+	LocalFetchBeacon     = "A Beacon result from a local Weaviate Local Fetch query"
+	LocalFetchClassName  = "The class name of the result from a local Weaviate Local Fetch query"
+	LocalFetchCertainty  = "The degree of similarity on a scale of 0-1 between the Beacon's characteristics and the provided concept"
+	LocalFetchActionsObj = "An object used to Fetch Beacons from the Actions subset of the dataset"
+)
+
+const (
+	LocalFetchFuzzyBeacon    = "A Beacon result from a local Weaviate Fetch Fuzzy query from both the Objects subsets"
+	LocalFetchFuzzyClassName = "Class name of the result from a local Weaviate Fetch Fuzzy query from both the Objects subsets"
+	LocalFetchFuzzyCertainty = "The degree of similarity on a scale of 0-1 between the Beacon's characteristics and the provided concept"
+	LocalFetchFuzzyObj       = "An object used to Fetch Beacons from both the Objects subsets"
+)
+
+// NETWORK
+const (
+	NetworkFetch    = "Fetch Beacons that are similar to a specified concept from the Objects subsets on a Weaviate network"
+	NetworkFetchObj = "An object used to perform a Fuzzy Fetch to search for Objects similar to a specified concept on a Weaviate network"
+)
+
+const (
+	NetworkFetchFuzzy = "Perform a Fuzzy Fetch to Fetch Beacons similar to a specified concept on a Weaviate network from both the Objects subsets"
+)
+
+const (
+	NetworkFetchFuzzyClassName = "The class name of the result from a network Weaviate Fetch Fuzzy query from both the Objects subsets"
+	NetworkFetchFuzzyBeacon    = "A Beacon result from a network Weaviate Fetch Fuzzy query from both the Objects subsets"
+	NetworkFetchFuzzyCertainty = "The degree of similarity on a scale of 0-1 between the Beacon's characteristics and the provided concept"
+	NetworkFetchFuzzyObj       = "An object used to Fetch Beacons from both the Objects subsets"
+)

adapters/handlers/graphql/descriptions/filters.go

@@ -0,0 +1,154 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// Where filter elements
+const (
+	GetWhere       = "Filter options for a local Get query, used to convert the result to the specified filters"
+	GetWhereInpObj = "An object containing filter options for a local Get query, used to convert the result to the specified filters"
+)
+
+const (
+	LocalMetaWhere       = "Filter options for a local Meta query, used to convert the result to the specified filters"
+	LocalMetaWhereInpObj = "An object containing filter options for a local Meta query, used to convert the result to the specified filters"
+)
+
+const (
+	AggregateWhere       = "Filter options for a local Aggregate query, used to convert the result to the specified filters"
+	AggregateWhereInpObj = "An object containing filter options for a local Aggregate query, used to convert the result to the specified filters"
+)
+
+const (
+	NetworkGetWhere       = "Filter options for a network Get query, used to convert the result to the specified filters"
+	NetworkGetWhereInpObj = "An object containing filter options for a network Get query, used to convert the result to the specified filters"
+)
+
+const (
+	NetworkMetaWhere       = "Filter options for a network Meta query, used to convert the result to the specified filters"
+	NetworkMetaWhereInpObj = "An object containing filter options for a network Meta query, used to convert the result to the specified filters"
+)
+
+const (
+	NetworkAggregateWhere       = "Filter options for a network Aggregate query, used to convert the result to the specified filters"
+	NetworkAggregateWhereInpObj = "An object containing filter options for a network Aggregate query, used to convert the result to the specified filters"
+)
+
+const (
+	WhereOperands       = "Contains the Operands that can be applied to a 'where' filter"
+	WhereOperandsInpObj = "An object containing the Operands that can be applied to a 'where' filter"
+)
+
+const (
+	WhereOperator     = "Contains the Operators that can be applied to a 'where' filter"
+	WhereOperatorEnum = "An object containing the Operators that can be applied to a 'where' filter"
+)
+
+const WherePath = "Specify the path from the Objects fields to the property name (e.g. ['Things', 'City', 'population'] leads to the 'population' property of a 'City' object)"
+
+const (
+	WhereValueInt                          = "Specify an Integer value that the target property will be compared to"
+	WhereValueNumber                       = "Specify a Float value that the target property will be compared to"
+	WhereValueBoolean                      = "Specify a Boolean value that the target property will be compared to"
+	WhereValueString                       = "Specify a String value that the target property will be compared to"
+	WhereValueRange                        = "Specify both geo-coordinates (latitude and longitude as decimals) and a maximum distance from the described coordinates. The search will return any result which is located less than or equal to the specified maximum distance in km away from the specified point."
+	WhereValueRangeGeoCoordinates          = "The geoCoordinates that form the center point of the search."
+	WhereValueRangeGeoCoordinatesLatitude  = "The latitude (in decimal format) of the geoCoordinates to search around."
+	WhereValueRangeGeoCoordinatesLongitude = "The longitude (in decimal format) of the geoCoordinates to search around."
+	WhereValueRangeDistance                = "The distance from the point specified via geoCoordinates."
+	WhereValueRangeDistanceMax             = "The maximum distance from the point specified geoCoordinates."
+	WhereValueText                         = "Specify a Text value that the target property will be compared to"
+	WhereValueDate                         = "Specify a Date value that the target property will be compared to"
+)
+
+// Properties and Classes filter elements (used by Fetch and Introspect Where filters)
+const (
+	WhereProperties    = "Specify which properties to filter on"
+	WherePropertiesObj = "Specify which properties to filter on"
+)
+
+const (
+	WherePropertiesPropertyName = "Specify which property name to filter properties on"
+	WhereCertainty              = "Specify the required degree of similarity between an object's characteristics and the provided filter values on a scale of 0-1"
+	WhereName                   = "Specify the name of the property to filter on"
+)
+
+const (
+	WhereKeywords       = "Specify which keywords to filter on"
+	WhereKeywordsInpObj = "Specify the value and the weight of a keyword"
+)
+
+const (
+	WhereKeywordsValue  = "Specify the value of the keyword"
+	WhereKeywordsWeight = "Specify the weight of the keyword"
+)
+
+const (
+	WhereClass  = "Specify which classes to filter on"
+	WhereInpObj = "Specify which classes and properties to filter on"
+)
+
+// Unique Fetch filter elements
+const (
+	FetchWhereFilterFields       = "An object containing filter options for a network Fetch search, used to convert the result to the specified filters"
+	FetchWhereFilterFieldsInpObj = "Filter options for a network Fetch search, used to convert the result to the specified filters"
+)
+
+const (
+	FetchFuzzyValue     = "Specify the concept that will be used to fetch Objects on the network (e.g. 'Airplane', or 'City')"
+	FetchFuzzyCertainty = "Specify how much a Beacon's characteristics must match the provided concept on a scale of 0 to 1"
+)
+
+// Unique Introspect filter elements
+const (
+	IntrospectWhereFilterFields       = "An object containing filter options for a network Fetch search, used to convert the result to the specified filters"
+	IntrospectWhereFilterFieldsInpObj = "Filter options for a network Fetch search, used to convert the result to the specified filters"
+	IntrospectBeaconId                = "The id of the Beacon"
+)
+
+// GroupBy filter elements
+const (
+	GroupByGroup  = "Specify the property of the class to group by"
+	GroupByCount  = "Get the number of instances of a property in a group"
+	GroupBySum    = "Get the sum of the values of a property in a group"
+	GroupByMin    = "Get the minimum occurring value of a property in a group"
+	GroupByMax    = "Get the maximum occurring value of a property in a group"
+	GroupByMean   = "Get the mean value of a property in a group"
+	GroupByMedian = "Get the median of a property in a group"
+	GroupByMode   = "Get the mode of a property in a group"
+)
+
+// Request timeout filter elements
+const NetworkTimeout = "Specify the time in seconds after which an unresolved request automatically fails"
+
+// Pagination filter elements
+const (
+	First = "Show the first x results (pagination option)"
+	After = "Show the results after the first x results (pagination option)"
+)
+
+// Cursor API
+const (
+	AfterID = "Show the results after a given ID"
+)
+
+const (
+	SortPath  = "Specify the path from the Objects fields to the property name (e.g. ['Get', 'City', 'population'] leads to the 'population' property of a 'City' object)"
+	SortOrder = "Specify the sort order, either ascending (asc) which is default or descending (desc)"
+)
+
+const (
+	GroupByFilter          = "Specify the property of the class to group by"
+	GroupByPath            = "Specify the path from the objects fields to the property name (e.g. ['Things', 'City', 'population'] leads to the 'population' property of a 'City' object)"
+	GroupByGroups          = "Specify the number of groups to be created"
+	GroupByObjectsPerGroup = "Specify the number of max objects in group"
+)

adapters/handlers/graphql/descriptions/get.go

@@ -0,0 +1,50 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// Local
+const (
+	GetObjects = "Get Objects on a local Weaviate"
+)
+
+const (
+	GetObj = "An object used to Get Objects on a local Weaviate"
+	Get    = "Get Objects on a local Weaviate"
+)
+
+const GetObjectsActionsObj = "An object used to get %ss on a local Weaviate"
+
+const GetClassUUID = "The UUID of a Object, assigned by its local Weaviate"
+
+// Network
+const (
+	NetworkGet    = "Get Objects from a Weaviate in a network"
+	NetworkGetObj = "An object used to Get Objects from a Weaviate in a network"
+)
+
+const NetworkGetWeaviateObj = "An object containing Get Objects fields for network Weaviate instance: "
+
+const (
+	NetworkGetObjects = "Get Objects from a Weaviate in a network"
+)
+
+const (
+	NetworkGetObjectsObj = "An object containing the Objects objects on this network Weaviate instance."
+)
+
+const NetworkGetClassUUID = "The UUID of a Object, assigned by the Weaviate network" // TODO check this with @lauraham
+
+const ConsistencyLevel = "Determines how many replicas must acknowledge a request " +
+	"before it is considered successful. Can be 'ONE', 'QUORUM', or 'ALL'"
+
+const Tenant = "The value by which a tenant is identified, specified in the class schema"

adapters/handlers/graphql/descriptions/getMeta.go

@@ -0,0 +1,88 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// Local
+const (
+	LocalMetaObj = "An object used to Get Meta information about Objects on a local Weaviate"
+	LocalMeta    = "Get Meta information about Objects on a local Weaviate"
+)
+
+const (
+	MetaPropertyType                 = "The datatype of this property"
+	MetaPropertyCount                = "The total amount of found instances for this property" // TODO check this with @lauraham
+	MetaPropertyTopOccurrences       = "An object containing data about the most frequently occurring values for this property"
+	MetaPropertyTopOccurrencesValue  = "The most frequently occurring value for this property"
+	MetaPropertyTopOccurrencesOccurs = "How often the most frequently occurring value for this property occurs" // TODO check this with @lauraham
+	MetaPropertyMinimum              = "The minimum value for this property"
+	MetaPropertyMaximum              = "The maximum value for this property"
+	MetaPropertyMean                 = "The mean of all values for this property"
+	MetaPropertySum                  = "The sum of all values for this property"
+	MetaPropertyObject               = "An object containing meta information about this property"
+)
+
+const (
+	AggregatePropertyType                 = "The datatype of this property"
+	AggregatePropertyCount                = "The total amount of found instances for this property" // TODO check this with @lauraham
+	AggregatePropertyTopOccurrences       = "An object containing data about the most frequently occurring values for this property"
+	AggregatePropertyTopOccurrencesValue  = "The most frequently occurring value for this property"
+	AggregatePropertyTopOccurrencesOccurs = "How often the most frequently occurring value for this property occurs" // TODO check this with @lauraham
+	AggregatePropertyMinimum              = "The minimum value for this property"
+	AggregatePropertyMaximum              = "The maximum value for this property"
+	AggregatePropertyMean                 = "The mean of all values for this property"
+	AggregatePropertySum                  = "The sum of all values for this property"
+)
+
+// Network
+const (
+	NetworkMeta            = "Get meta information about Objects from a Weaviate in a network"
+	NetworkMetaObj         = "An object used to Get meta information about Objects from a Weaviate in a network"
+	NetworkMetaWeaviateObj = "An object containing the Meta Objects fields for network Weaviate instance: "
+)
+
+const (
+	MetaMetaProperty = "Meta information about the object"
+	MetaProperty     = "Meta information about the property "
+)
+
+const (
+	MetaClassPropertyTotalTrue      = "How often this boolean property's value is true in the dataset"
+	MetaClassPropertyPercentageTrue = "The percentage of true values for this boolean property in the dataset"
+)
+
+const (
+	MetaClassPropertyTotalFalse      = "How often this boolean property's value is false in the dataset"
+	MetaClassPropertyPercentageFalse = "The percentage of false values for this boolean property in the dataset"
+)
+
+const (
+	MetaClassPropertyPointingTo = "The classes that this object contains a reference to"
+	MetaClassMetaCount          = "The total amount of found instances for a class"
+	MetaClassMetaObj            = "An object containing Meta information about a class"
+)
+
+const (
+	AggregateClassPropertyTotalTrue      = "How often this boolean property's value is true in the dataset"
+	AggregateClassPropertyPercentageTrue = "The percentage of true values for this boolean property in the dataset"
+)
+
+const (
+	AggregateClassPropertyTotalFalse      = "How often this boolean property's value is false in the dataset"
+	AggregateClassPropertyPercentageFalse = "The percentage of false values for this boolean property in the dataset"
+)
+
+const (
+	AggregateClassPropertyPointingTo = "The classes that this object contains a reference to"
+	AggregateClassAggregateCount     = "The total amount of found instances for a class"
+	AggregateClassAggregateObj       = "An object containing Aggregate information about a class"
+)

adapters/handlers/graphql/descriptions/introspect.go

@@ -0,0 +1,30 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// NETWORK
+const (
+	NetworkIntrospect    = "Get Introspection information about Objects and/or Beacons in a Weaviate network"
+	NetworkIntrospectObj = "An object used to perform an Introspection query on a Weaviate network"
+)
+
+const (
+	NetworkIntrospectWeaviate  = "The Weaviate instance that the current Object or Beacon belongs to"
+	NetworkIntrospectClassName = "The name of the current Object or Beacon's class"
+	NetworkIntrospectCertainty = "The degree of similarity between a(n) Object or Beacon and the filter input"
+)
+
+const (
+	NetworkIntrospectBeaconProperties             = "The properties of a Beacon"
+	NetworkIntrospectBeaconPropertiesPropertyName = "The names of the properties of a Beacon"
+)

adapters/handlers/graphql/descriptions/merge.go

@@ -0,0 +1,31 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// Local
+const (
+	LocalMergeObj = "An object used to Merge Objects on a local Weaviate"
+	LocalMerge    = "Merge Objects on a local Weaviate"
+)
+
+const LocalMergeClassUUID = "The UUID of a Object, assigned by its local Weaviate"
+
+// Network
+const (
+	NetworkMerge    = "Merge Objects from a Weaviate in a network"
+	NetworkMergeObj = "An object used to Merge Objects from a Weaviate in a network"
+)
+
+const NetworkMergeWeaviateObj = "An object containing Merge Objects fields for network Weaviate instance: "
+
+const NetworkMergeClassUUID = "The UUID of a Thing or Action, assigned by the Weaviate network" // TODO check this with @lauraham

adapters/handlers/graphql/descriptions/rootQuery.go

@@ -0,0 +1,28 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate
+package descriptions
+
+// ROOT
+const (
+	WeaviateObj     = "The location of the root query"
+	WeaviateNetwork = "Query a Weaviate network"
+)
+
+// LOCAL
+const LocalObj = "A query on a local Weaviate"
+
+// NETWORK
+const (
+	NetworkWeaviate = "An object for the network Weaviate instance: "
+	NetworkObj      = "An object used to perform queries on a Weaviate network"
+)

adapters/handlers/graphql/graphiql/graphiql.go

@@ -0,0 +1,233 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+// Based on `graphiql.go` from https://github.com/graphql-go/handler
+// only made RenderGraphiQL a public function.
+package graphiql
+
+import (
+	"encoding/json"
+	"html/template"
+	"net/http"
+	"strings"
+)
+
+// graphiqlVersion is the current version of GraphiQL
+const graphiqlVersion = "0.11.11"
+
+// graphiqlData is the page data structure of the rendered GraphiQL page
+type graphiqlData struct {
+	GraphiqlVersion string
+	QueryString     string
+	Variables       string
+	OperationName   string
+	AuthKey         string
+	AuthToken       string
+}
+
+func AddMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/v1/graphql") && r.Method == http.MethodGet {
+			renderGraphiQL(w, r)
+		} else {
+			next.ServeHTTP(w, r)
+		}
+	})
+}
+
+// renderGraphiQL renders the GraphiQL GUI
+func renderGraphiQL(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("WWW-Authenticate", `Basic realm="Provide your key and token (as username as password respectively)"`)
+
+	user, password, authOk := r.BasicAuth()
+	if !authOk {
+		http.Error(w, "Not authorized", 401)
+		return
+	}
+
+	queryParams := r.URL.Query()
+
+	t := template.New("GraphiQL")
+	t, err := t.Parse(graphiqlTemplate)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Attempt to deserialize the 'variables' query key to something reasonable.
+	var queryVars interface{}
+	err = json.Unmarshal([]byte(queryParams.Get("variables")), &queryVars)
+
+	var varsString string
+	if err == nil {
+		vars, err := json.MarshalIndent(queryVars, "", "  ")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		varsString = string(vars)
+		if varsString == "null" {
+			varsString = ""
+		}
+	}
+
+	// Create result string
+	d := graphiqlData{
+		GraphiqlVersion: graphiqlVersion,
+		QueryString:     queryParams.Get("query"),
+		Variables:       varsString,
+		OperationName:   queryParams.Get("operationName"),
+		AuthKey:         user,
+		AuthToken:       password,
+	}
+	err = t.ExecuteTemplate(w, "index", d)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+// tmpl is the page template to render GraphiQL
+const graphiqlTemplate = `
+{{ define "index" }}
+<!--
+The request to this GraphQL server provided the header "Accept: text/html"
+and as a result has been presented GraphiQL - an in-browser IDE for
+exploring GraphQL.
+
+If you wish to receive JSON, provide the header "Accept: application/json" or
+add "&raw" to the end of the URL within a browser.
+-->
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8" />
+  <title>GraphiQL</title>
+  <meta name="robots" content="noindex" />
+  <meta name="referrer" content="origin">
+  <style>
+    body {
+      height: 100%;
+      margin: 0;
+      overflow: hidden;
+      width: 100%;
+    }
+    #graphiql {
+      height: 100vh;
+    }
+  </style>
+  <link href="//cdn.jsdelivr.net/npm/graphiql@{{ .GraphiqlVersion }}/graphiql.css" rel="stylesheet" />
+  <script src="//cdn.jsdelivr.net/es6-promise/4.0.5/es6-promise.auto.min.js"></script>
+  <script src="//cdn.jsdelivr.net/fetch/0.9.0/fetch.min.js"></script>
+  <script src="//cdn.jsdelivr.net/react/15.4.2/react.min.js"></script>
+  <script src="//cdn.jsdelivr.net/react/15.4.2/react-dom.min.js"></script>
+  <script src="//cdn.jsdelivr.net/npm/graphiql@{{ .GraphiqlVersion }}/graphiql.min.js"></script>
+</head>
+<body>
+  <div id="graphiql">Loading...</div>
+  <script>
+    // Collect the URL parameters
+    var parameters = {};
+    window.location.search.substr(1).split('&').forEach(function (entry) {
+      var eq = entry.indexOf('=');
+      if (eq >= 0) {
+        parameters[decodeURIComponent(entry.slice(0, eq))] =
+          decodeURIComponent(entry.slice(eq + 1));
+      }
+    });
+
+    // Produce a Location query string from a parameter object.
+    function locationQuery(params) {
+      return '?' + Object.keys(params).filter(function (key) {
+        return Boolean(params[key]);
+      }).map(function (key) {
+        return encodeURIComponent(key) + '=' +
+          encodeURIComponent(params[key]);
+      }).join('&');
+    }
+
+    // Derive a fetch URL from the current URL, sans the GraphQL parameters.
+    var graphqlParamNames = {
+      query: true,
+      variables: true,
+      operationName: true
+    };
+
+    var otherParams = {};
+    for (var k in parameters) {
+      if (parameters.hasOwnProperty(k) && graphqlParamNames[k] !== true) {
+        otherParams[k] = parameters[k];
+      }
+    }
+    var fetchURL = locationQuery(otherParams);
+
+    // Defines a GraphQL fetcher using the fetch API.
+    function graphQLFetcher(graphQLParams) {
+      return fetch(fetchURL, {
+        method: 'post',
+        headers: {
+          'Accept': 'application/json',
+          'Content-Type': 'application/json',
+          'X-API-KEY': {{ .AuthKey }},
+          'X-API-TOKEN': {{ .AuthToken }}
+        },
+        body: JSON.stringify(graphQLParams),
+        credentials: 'include',
+      }).then(function (response) {
+        return response.text();
+      }).then(function (responseBody) {
+        try {
+          return JSON.parse(responseBody);
+        } catch (error) {
+          return responseBody;
+        }
+      });
+    }
+
+    // When the query and variables string is edited, update the URL bar so
+    // that it can be easily shared.
+    function onEditQuery(newQuery) {
+      parameters.query = newQuery;
+      updateURL();
+    }
+
+    function onEditVariables(newVariables) {
+      parameters.variables = newVariables;
+      updateURL();
+    }
+
+    function onEditOperationName(newOperationName) {
+      parameters.operationName = newOperationName;
+      updateURL();
+    }
+
+    function updateURL() {
+      history.replaceState(null, null, locationQuery(parameters));
+    }
+
+    // Render <GraphiQL /> into the body.
+    ReactDOM.render(
+      React.createElement(GraphiQL, {
+        fetcher: graphQLFetcher,
+        onEditQuery: onEditQuery,
+        onEditVariables: onEditVariables,
+        onEditOperationName: onEditOperationName,
+        query: {{ .QueryString }},
+        response: null,
+        variables: {{ .Variables }},
+        operationName: {{ .OperationName }},
+      }),
+      document.getElementById('graphiql')
+    );
+  </script>
+</body>
+</html>
+{{ end }}
+`

adapters/handlers/graphql/local/aggregate/aggregate.go

@@ -0,0 +1,281 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package aggregate
+
+import (
+	"fmt"
+
+	"github.com/tailor-inc/graphql"
+	"github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions"
+	"github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters"
+	"github.com/weaviate/weaviate/adapters/handlers/graphql/utils"
+	"github.com/weaviate/weaviate/entities/aggregation"
+	"github.com/weaviate/weaviate/entities/models"
+	"github.com/weaviate/weaviate/entities/schema"
+	"github.com/weaviate/weaviate/usecases/config"
+)
+
+type ModulesProvider interface {
+	AggregateArguments(class *models.Class) map[string]*graphql.ArgumentConfig
+	ExtractSearchParams(arguments map[string]interface{}, className string) map[string]interface{}
+}
+
+// Build the Aggregate Kinds schema
+func Build(dbSchema *schema.Schema, config config.Config,
+	modulesProvider ModulesProvider,
+) (*graphql.Field, error) {
+	if len(dbSchema.Objects.Classes) == 0 {
+		return nil, utils.ErrEmptySchema
+	}
+
+	var err error
+	var localAggregateObjects *graphql.Object
+	if len(dbSchema.Objects.Classes) > 0 {
+		localAggregateObjects, err = classFields(dbSchema.Objects.Classes, config, modulesProvider)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	field := graphql.Field{
+		Name:        "Aggregate",
+		Description: descriptions.AggregateWhere,
+		Type:        localAggregateObjects,
+		Resolve:     passThroughResolver,
+	}
+
+	return &field, nil
+}
+
+func classFields(databaseSchema []*models.Class,
+	config config.Config, modulesProvider ModulesProvider,
+) (*graphql.Object, error) {
+	fields := graphql.Fields{}
+
+	for _, class := range databaseSchema {
+		field, err := classField(class, class.Description, config, modulesProvider)
+		if err != nil {
+			return nil, err
+		}
+
+		fields[class.Class] = field
+	}
+
+	return graphql.NewObject(graphql.ObjectConfig{
+		Name:        "AggregateObjectsObj",
+		Fields:      fields,
+		Description: descriptions.AggregateObjectsObj,
+	}), nil
+}
+
+func classField(class *models.Class, description string,
+	config config.Config, modulesProvider ModulesProvider,
+) (*graphql.Field, error) {
+	metaClassName := fmt.Sprintf("Aggregate%s", class.Class)
+
+	fields := graphql.ObjectConfig{
+		Name: metaClassName,
+		Fields: (graphql.FieldsThunk)(func() graphql.Fields {
+			fields, err := classPropertyFields(class)
+			if err != nil {
+				// we cannot return an error in this FieldsThunk and have to panic unfortunately
+				panic(fmt.Sprintf("Failed to assemble single Local Aggregate Class field: %s", err))
+			}
+
+			return fields
+		}),
+		Description: description,
+	}
+
+	fieldsObject := graphql.NewObject(fields)
+	fieldsField := &graphql.Field{
+		Type:        graphql.NewList(fieldsObject),
+		Description: description,
+		Args: graphql.FieldConfigArgument{
+			"limit": &graphql.ArgumentConfig{
+				Description: descriptions.First,
+				Type:        graphql.Int,
+			},
+			"where": &graphql.ArgumentConfig{
+				Description: descriptions.GetWhere,
+				Type: graphql.NewInputObject(
+					graphql.InputObjectConfig{
+						Name:        fmt.Sprintf("AggregateObjects%sWhereInpObj", class.Class),
+						Fields:      common_filters.BuildNew(fmt.Sprintf("AggregateObjects%s", class.Class)),
+						Description: descriptions.GetWhereInpObj,
+					},
+				),
+			},
+			"groupBy": &graphql.ArgumentConfig{
+				Description: descriptions.GroupBy,
+				Type:        graphql.NewList(graphql.String),
+			},
+			"nearVector": nearVectorArgument(class.Class),
+			"nearObject": nearObjectArgument(class.Class),
+			"objectLimit": &graphql.ArgumentConfig{
+				Description: descriptions.First,
+				Type:        graphql.Int,
+			},
+			"hybrid": hybridArgument(fieldsObject, class, modulesProvider),
+		},
+		Resolve: makeResolveClass(modulesProvider, class),
+	}
+
+	if modulesProvider != nil {
+		for name, argument := range modulesProvider.AggregateArguments(class) {
+			fieldsField.Args[name] = argument
+		}
+	}
+
+	if schema.MultiTenancyEnabled(class) {
+		fieldsField.Args["tenant"] = tenantArgument()
+	}
+
+	return fieldsField, nil
+}
+
+func classPropertyFields(class *models.Class) (graphql.Fields, error) {
+	fields := graphql.Fields{}
+	for _, property := range class.Properties {
+		propertyType, err := schema.GetPropertyDataType(class, property.Name)
+		if err != nil {
+			return nil, fmt.Errorf("%s.%s: %s", class.Class, property.Name, err)
+		}
+
+		convertedDataType, err := classPropertyField(*propertyType, class, property)
+		if err != nil {
+			return nil, err
+		}
+
+		fields[property.Name] = convertedDataType
+	}
+
+	// Special case: meta { count } appended to all regular props
+	fields["meta"] = &graphql.Field{
+		Description: descriptions.LocalMetaObj,
+		Type:        metaObject(fmt.Sprintf("Aggregate%s", class.Class)),
+		Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+			// pass-through
+			return p.Source, nil
+		},
+	}
+
+	// Always append Grouped By field
+	fields["groupedBy"] = &graphql.Field{
+		Description: descriptions.AggregateGroupedBy,
+		Type:        groupedByProperty(class),
+		Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+			switch typed := p.Source.(type) {
+			case aggregation.Group:
+				return typed.GroupedBy, nil
+			case map[string]interface{}:
+				return typed["groupedBy"], nil
+			default:
+				return nil, fmt.Errorf("groupedBy: unsupported type %T", p.Source)
+			}
+		},
+	}
+
+	return fields, nil
+}
+
+func metaObject(prefix string) *graphql.Object {
+	return graphql.NewObject(graphql.ObjectConfig{
+		Name: fmt.Sprintf("%sMetaObject", prefix),
+		Fields: graphql.Fields{
+			"count": &graphql.Field{
+				Type: graphql.Int,
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					group, ok := p.Source.(aggregation.Group)
+					if !ok {
+						return nil, fmt.Errorf("meta count: expected aggregation.Group, got %T", p.Source)
+					}
+
+					return group.Count, nil
+				},
+			},
+		},
+	})
+}
+
+func classPropertyField(dataType schema.DataType, class *models.Class, property *models.Property) (*graphql.Field, error) {
+	switch dataType {
+	case schema.DataTypeText:
+		return makePropertyField(class, property, stringPropertyFields)
+	case schema.DataTypeInt:
+		return makePropertyField(class, property, numericPropertyFields)
+	case schema.DataTypeNumber:
+		return makePropertyField(class, property, numericPropertyFields)
+	case schema.DataTypeBoolean:
+		return makePropertyField(class, property, booleanPropertyFields)
+	case schema.DataTypeDate:
+		return makePropertyField(class, property, datePropertyFields)
+	case schema.DataTypeCRef:
+		return makePropertyField(class, property, referencePropertyFields)
+	case schema.DataTypeGeoCoordinates:
+		// simply skip for now, see gh-729
+		return nil, nil
+	case schema.DataTypePhoneNumber:
+		// skipping for now, see gh-1088 where it was outscoped
+		return nil, nil
+	case schema.DataTypeBlob:
+		return makePropertyField(class, property, stringPropertyFields)
+	case schema.DataTypeTextArray:
+		return makePropertyField(class, property, stringPropertyFields)
+	case schema.DataTypeIntArray, schema.DataTypeNumberArray:
+		return makePropertyField(class, property, numericPropertyFields)
+	case schema.DataTypeBooleanArray:
+		return makePropertyField(class, property, booleanPropertyFields)
+	case schema.DataTypeDateArray:
+		return makePropertyField(class, property, datePropertyFields)
+	case schema.DataTypeUUID, schema.DataTypeUUIDArray:
+		// not aggregatable
+		return nil, nil
+	case schema.DataTypeObject, schema.DataTypeObjectArray:
+		// TODO: check if it's aggregable, skip for now
+		return nil, nil
+	default:
+		return nil, fmt.Errorf(schema.ErrorNoSuchDatatype+": %s", dataType)
+	}
+}
+
+type propertyFieldMaker func(class *models.Class,
+	property *models.Property, prefix string) *graphql.Object
+
+func makePropertyField(class *models.Class, property *models.Property,
+	fieldMaker propertyFieldMaker,
+) (*graphql.Field, error) {
+	prefix := "Aggregate"
+	return &graphql.Field{
+		Description: fmt.Sprintf(`%s"%s"`, descriptions.AggregateProperty, property.Name),
+		Type:        fieldMaker(class, property, prefix),
+		Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+			switch typed := p.Source.(type) {
+			case aggregation.Group:
+				res, ok := typed.Properties[property.Name]
+				if !ok {
+					return nil, fmt.Errorf("missing property '%s'", property.Name)
+				}
+
+				return res, nil
+
+			default:
+				return nil, fmt.Errorf("property %s, unsupported type %T", property.Name, p.Source)
+			}
+		},
+	}, nil
+}
+
+func passThroughResolver(p graphql.ResolveParams) (interface{}, error) {
+	// bubble up root resolver
+	return p.Source, nil
+}

adapters/handlers/graphql/local/aggregate/explore_argument.go

@@ -0,0 +1,25 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package aggregate
+
+import (
+	"github.com/tailor-inc/graphql"
+	"github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters"
+)
+
+func nearVectorArgument(className string) *graphql.ArgumentConfig {
+	return common_filters.NearVectorArgument("AggregateObjects", className)
+}
+
+func nearObjectArgument(className string) *graphql.ArgumentConfig {
+	return common_filters.NearObjectArgument("AggregateObjects", className)
+}

adapters/handlers/graphql/local/aggregate/helpers_for_test.go

@@ -0,0 +1,56 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package aggregate
+
+import (
+	"context"
+	"fmt"
+
+	testhelper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper"
+	"github.com/weaviate/weaviate/entities/aggregation"
+	"github.com/weaviate/weaviate/entities/models"
+	"github.com/weaviate/weaviate/usecases/config"
+)
+
+type mockRequestsLog struct{}
+
+func (m *mockRequestsLog) Register(first string, second string) {
+}
+
+type mockResolver struct {
+	testhelper.MockResolver
+}
+
+func newMockResolver(cfg config.Config) *mockResolver {
+	field, err := Build(&testhelper.CarSchema, cfg, nil)
+	if err != nil {
+		panic(fmt.Sprintf("could not build graphql test schema: %s", err))
+	}
+	mockLog := &mockRequestsLog{}
+	mocker := &mockResolver{}
+	mocker.RootFieldName = "Aggregate"
+	mocker.RootField = field
+	mocker.RootObject = map[string]interface{}{
+		"Resolver":    Resolver(mocker),
+		"RequestsLog": mockLog,
+		"Config":      cfg,
+	}
+
+	return mocker
+}
+
+func (m *mockResolver) Aggregate(ctx context.Context, principal *models.Principal,
+	params *aggregation.Params,
+) (interface{}, error) {
+	args := m.Called(params)
+	return args.Get(0), args.Error(1)
+}

adapters/handlers/graphql/local/aggregate/hybrid_search.go

@@ -0,0 +1,90 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: [email protected]
+//
+
+package aggregate
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/tailor-inc/graphql"
+	"github.com/weaviate/weaviate/entities/models"
+)
+
+func hybridArgument(classObject *graphql.Object,
+	class *models.Class, modulesProvider ModulesProvider,
+) *graphql.ArgumentConfig {
+	prefix := fmt.Sprintf("AggregateObjects%s", class.Class)
+	return &graphql.ArgumentConfig{
+		Type: graphql.NewInputObject(
+			graphql.InputObjectConfig{
+				Name:        fmt.Sprintf("%sHybridInpObj", prefix),
+				Fields:      hybridOperands(classObject, class, modulesProvider),
+				Description: "Hybrid search",
+			},
+		),
+	}
+}
+
+func hybridOperands(classObject *graphql.Object,
+	class *models.Class, modulesProvider ModulesProvider,
+) graphql.InputObjectConfigFieldMap {
+	ss := graphql.NewInputObject(graphql.InputObjectConfig{
+		Name:   class.Class + "HybridSubSearch",
+		Fields: hybridSubSearch(classObject, class, modulesProvider),
+	})
+	fieldMap := graphql.InputObjectConfigFieldMap{
+		"query": &graphql.InputObjectFieldConfig{
+			Description: "Query string",
+			Type:        graphql.String,
+		},
+		"alpha": &graphql.InputObjectFieldConfig{
+			Description: "Search weight",
+			Type:        graphql.Float,
+		},
+		"vector": &graphql.InputObjectFieldConfig{
+			Description: "Vector search",
+			Type:        graphql.NewList(graphql.Float),
+		},
+	}
+
+	if os.Getenv("ENABLE_EXPERIMENTAL_HYBRID_OPERANDS") != "" {
+		fieldMap["operands"] = &graphql.InputObjectFieldConfig{
+			Description: "Subsearch list",
+			Type:        graphql.NewList(ss),
+		}
+	}
+
+	return fieldMap
+}
+
+func hybridSubSearch(classObject *graphql.Object,
+	class *models.Class, modulesProvider ModulesProvider,
+) graphql.InputObjectConfigFieldMap {
+	prefixName := class.Class + "SubSearch"
+
+	return graphql.InputObjectConfigFieldMap{
+		"weight": &graphql.InputObjectFieldConfig{
+			Description: "weight, 0 to 1",
+			Type:        graphql.Float,
+		},
+		"sparseSearch": &graphql.InputObjectFieldConfig{
+			Description: "Sparse Search",
+			Type: graphql.NewInputObject(
+				graphql.InputObjectConfig{
+					Name:        fmt.Sprintf("%sHybridAggregateBM25InpObj", prefixName),
+					Fields:      bm25Fields(prefixName),
+					Description: "BM25f search",
+				},
+			),
+		},
+	}
+}